You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by kr...@apache.org on 2025/01/23 10:03:18 UTC
(hive) branch master updated: HIVE-28572: Support Distribute by and Cluster by clauses in CBO (Krisztian Kasa, reviewed by Stamatis Zampetakis, Soumyakanti Das)
This is an automated email from the ASF dual-hosted git repository.
krisztiankasa pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 43f89ff690a HIVE-28572: Support Distribute by and Cluster by clauses in CBO (Krisztian Kasa, reviewed by Stamatis Zampetakis, Soumyakanti Das)
43f89ff690a is described below
commit 43f89ff690a9322ae37ef79861c57ba68b0d5883
Author: Krisztian Kasa <kk...@cloudera.com>
AuthorDate: Thu Jan 23 11:03:11 2025 +0100
HIVE-28572: Support Distribute by and Cluster by clauses in CBO (Krisztian Kasa, reviewed by Stamatis Zampetakis, Soumyakanti Das)
---
.../calcite/rules/HiveFilterSortTransposeRule.java | 31 +-
.../HiveProjectSortExchangeTransposeRule.java | 5 +
.../optimizer/calcite/translator/ASTConverter.java | 45 +-
.../opconventer/HiveOpConverterUtils.java | 10 +-
.../opconventer/HiveSortExchangeVisitor.java | 54 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 598 +++++++++++----------
.../test/queries/clientnegative/qualify_no_cbo_2.q | 8 -
.../cbo_distribute_sort_cluster_by.q | 21 +
.../cbo_rp_distribute_sort_cluster_by.q | 15 +
.../results/clientnegative/qualify_no_cbo_2.q.out | 9 -
.../llap/cbo_distribute_sort_cluster_by.q.out | 234 ++++++++
.../llap/cbo_rp_distribute_sort_cluster_by.q.out | 181 +++++++
.../test/results/clientpositive/llap/cluster.q.out | 428 ++++++++-------
.../llap/columnarserde_create_shortcut.q.out | 2 +-
.../llap/correlationoptimizer14.q.out | 76 +--
.../llap/dynpart_sort_optimization2.q.out | 24 +-
.../llap/groupingset_optimize_hive_28489.q.out | 14 +-
.../llap/implicit_cast_during_insert.q.out | 42 +-
.../llap/infer_bucket_sort_num_buckets.q.out | 18 +-
.../clientpositive/llap/input_columnarserde.q.out | 2 +-
.../clientpositive/llap/input_lazyserde.q.out | 2 +-
.../clientpositive/llap/input_lazyserde2.q.out | 2 +-
.../results/clientpositive/llap/mapreduce5.q.out | 2 +-
.../results/clientpositive/llap/mapreduce6.q.out | 2 +-
ql/src/test/results/clientpositive/llap/ppd2.q.out | 127 ++---
.../clientpositive/llap/ppd_clusterby.q.out | 183 ++++---
.../llap/reduce_deduplicate_exclude_join.q.out | 24 +-
.../llap/reduce_deduplicate_stat_based.q.out | 24 +-
.../clientpositive/llap/reducesink_dedup.q.out | 34 +-
29 files changed, 1378 insertions(+), 839 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortTransposeRule.java
index bdd7b0da5b8..97603ab9297 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterSortTransposeRule.java
@@ -22,50 +22,49 @@
import org.apache.calcite.rel.RelNode;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortExchange;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSortLimit;
import com.google.common.collect.ImmutableList;
+import static java.util.Collections.singletonList;
+
public class HiveFilterSortTransposeRule extends RelOptRule {
- public static final HiveFilterSortTransposeRule INSTANCE =
- new HiveFilterSortTransposeRule();
+ public static final HiveFilterSortTransposeRule SORT_LIMIT_INSTANCE =
+ new HiveFilterSortTransposeRule(HiveSortLimit.class);
+ public static final HiveFilterSortTransposeRule SORT_EXCHANGE_INSTANCE =
+ new HiveFilterSortTransposeRule(HiveSortExchange.class);
//~ Constructors -----------------------------------------------------------
/**
* Creates a HiveFilterSortTransposeRule.
*/
- private HiveFilterSortTransposeRule() {
+ private HiveFilterSortTransposeRule(Class<? extends RelNode> clazz) {
super(
operand(
HiveFilter.class,
- operand(HiveSortLimit.class, any())));
+ operand(clazz, any())));
}
//~ Methods ----------------------------------------------------------------
public boolean matches(RelOptRuleCall call) {
- final HiveSortLimit sort = call.rel(1);
+ final RelNode sort = call.rel(1);
// If sort contains a limit operation, we bail out
- if (HiveCalciteUtil.limitRelNode(sort)) {
- return false;
- }
-
- return true;
+ return !HiveCalciteUtil.limitRelNode(sort);
}
public void onMatch(RelOptRuleCall call) {
final HiveFilter filter = call.rel(0);
- final HiveSortLimit sort = call.rel(1);
+ final RelNode sort = call.rel(1);
- final RelNode newFilter = filter.copy(sort.getInput().getTraitSet(),
- ImmutableList.<RelNode>of(sort.getInput()));
- final HiveSortLimit newSort = sort.copy(sort.getTraitSet(),
- newFilter, sort.collation, sort.offset, sort.fetch);
+ final RelNode newFilter = filter.copy(sort.getInput(0).getTraitSet(),
+ ImmutableList.of(sort.getInput(0)));
+ final RelNode newSort = sort.copy(sort.getTraitSet(), singletonList(newFilter));
call.transformTo(newSort);
}
-
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
index cd92b86a4c2..a1fbdbcba93 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveProjectSortExchangeTransposeRule.java
@@ -88,6 +88,11 @@ public void onMatch(RelOptRuleCall call) {
RelCollation newCollation = RelCollationTraitDef.INSTANCE.canonize(RelCollationImpl.of(fieldCollations));
List<Integer> newDistributionKeys = getNewRelDistributionKeys(project, sortExchange.getDistribution());
+ if (newDistributionKeys == null) {
+ // Project can not be pushed because it does not project necessary expressions referenced by distribution keys
+ return;
+ }
+
RelDistribution newDistribution = RelDistributionTraitDef.INSTANCE.canonize(
new HiveRelDistribution(sortExchange.getDistribution().getType(), newDistributionKeys));
RelTraitSet newTraitSet = TraitsUtil.getDefaultTraitSet(sortExchange.getCluster())
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
index f2823dfcc67..4819138bcec 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
@@ -30,6 +30,7 @@
import org.apache.calcite.adapter.druid.DruidQuery;
import org.apache.calcite.plan.RelOptUtil;
+import org.apache.calcite.rel.RelDistribution;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.RelVisitor;
@@ -452,8 +453,17 @@ private void convertOrderLimitToASTNode(HiveSortLimit hiveSortLimit) {
}
private void convertSortToASTNode(HiveSortExchange hiveSortExchange) {
+ Schema sortExchangeSchema = new Schema(hiveSortExchange);
+ if (hiveSortExchange.getDistribution().getType() != RelDistribution.Type.ANY) {
+ ASTNode distributeByAst = ASTBuilder.createAST(HiveParser.TOK_DISTRIBUTEBY, "TOK_DISTRIBUTEBY");
+ for (int i : hiveSortExchange.getDistribution().getKeys()) {
+ distributeByAst.addChild(
+ convertSortKey(hiveSortExchange, sortExchangeSchema, null, sortExchangeSchema.get(i)));
+ }
+ hiveAST.distributeBy = distributeByAst;
+ }
List<RelFieldCollation> fieldCollations = hiveSortExchange.getCollation().getFieldCollations();
- convertFieldCollationsToASTNode(hiveSortExchange, new Schema(hiveSortExchange), fieldCollations,
+ convertFieldCollationsToASTNode(hiveSortExchange, sortExchangeSchema, fieldCollations,
null, HiveParser.TOK_SORTBY, "TOK_SORTBY");
}
@@ -467,8 +477,6 @@ private void convertFieldCollationsToASTNode(
// 1 Add order/sort by token
ASTNode orderAst = ASTBuilder.createAST(astToken, astText);
- RexNode obExpr;
- ASTNode astCol;
for (RelFieldCollation c : fieldCollations) {
// 2 Add Direction token
@@ -496,21 +504,11 @@ private void convertFieldCollationsToASTNode(
// 3 Convert OB expr (OB Expr is usually an input ref except for top
// level OB; top level OB will have RexCall kept in a map.)
- obExpr = null;
+ RexNode obExpr = null;
if (obRefToCallMap != null) {
obExpr = obRefToCallMap.get(c.getFieldIndex());
}
-
- if (obExpr != null) {
- astCol = obExpr.accept(new RexVisitor(schema, false, node.getCluster().getRexBuilder()));
- } else {
- ColumnInfo cI = schema.get(c.getFieldIndex());
- /*
- * The RowResolver setup for Select drops Table associations. So
- * setup ASTNode on unqualified name.
- */
- astCol = ASTBuilder.unqualifiedName(cI.column);
- }
+ ASTNode astCol = convertSortKey(node, schema, obExpr, schema.get(c.getFieldIndex()));
// 4 buildup the ob expr AST
nullDirectionAST.addChild(astCol);
@@ -519,6 +517,20 @@ private void convertFieldCollationsToASTNode(
hiveAST.order = orderAst;
}
+ private ASTNode convertSortKey(RelNode node, Schema schema, RexNode obExpr, ColumnInfo columnInfo) {
+ ASTNode astCol;
+ if (obExpr != null) {
+ astCol = obExpr.accept(new RexVisitor(schema, false, node.getCluster().getRexBuilder()));
+ } else {
+ /*
+ * The RowResolver setup for Select drops Table associations. So
+ * setup ASTNode on unqualified name.
+ */
+ astCol = ASTBuilder.unqualifiedName(columnInfo.column);
+ }
+ return astCol;
+ }
+
private Schema getRowSchema(String tblAlias) {
if (select instanceof Project) {
return new Schema((Project) select, tblAlias);
@@ -1267,6 +1279,7 @@ static class HiveAST {
ASTNode groupBy;
ASTNode having;
ASTNode select;
+ ASTNode distributeBy;
ASTNode order;
ASTNode limit;
@@ -1276,7 +1289,7 @@ public ASTNode getAST() {
.add(from)
.add(
ASTBuilder.construct(HiveParser.TOK_INSERT, "TOK_INSERT").add(ASTBuilder.destNode())
- .add(select).add(where).add(groupBy).add(having).add(order).add(limit));
+ .add(select).add(where).add(groupBy).add(having).add(distributeBy).add(order).add(limit));
return b.node();
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveOpConverterUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveOpConverterUtils.java
index 55130b9a5be..ae1c7670298 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveOpConverterUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveOpConverterUtils.java
@@ -112,7 +112,7 @@ static SelectOperator genReduceSinkAndBacktrackSelect(Operator<?> input,
@SuppressWarnings({ "rawtypes", "unchecked" })
static ReduceSinkOperator genReduceSink(Operator<?> input, String tableAlias, ExprNodeDesc[] keys, int tag,
- ArrayList<ExprNodeDesc> partitionCols, String order, String nullOrder, int numReducers,
+ List<ExprNodeDesc> partitionCols, String order, String nullOrder, int numReducers,
Operation acidOperation, HiveConf hiveConf) throws SemanticException {
Operator dummy = Operator.createDummy(); // dummy for backtracking
dummy.setParentOperators(Arrays.asList(input));
@@ -183,14 +183,8 @@ static ReduceSinkOperator genReduceSink(Operator<?> input, String tableAlias, Ex
}
}
- ReduceSinkDesc rsDesc;
- if (order.isEmpty()) {
- rsDesc = PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, outputColumnNames, false, tag,
- reduceKeys.size(), numReducers, acidOperation, NullOrdering.defaultNullOrder(hiveConf));
- } else {
- rsDesc = PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, outputColumnNames, false, tag,
+ ReduceSinkDesc rsDesc = PlanUtils.getReduceSinkDesc(reduceKeys, reduceValues, outputColumnNames, false, tag,
partitionCols, order, nullOrder, NullOrdering.defaultNullOrder(hiveConf), numReducers, acidOperation, false);
- }
ReduceSinkOperator rsOp = (ReduceSinkOperator) OperatorFactory.getAndMakeChild(
rsDesc, new RowSchema(outputColumns), input);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveSortExchangeVisitor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveSortExchangeVisitor.java
index 8a784c8727f..b8242cfc500 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveSortExchangeVisitor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/opconventer/HiveSortExchangeVisitor.java
@@ -19,10 +19,13 @@
package org.apache.hadoop.hive.ql.optimizer.calcite.translator.opconventer;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
import org.apache.calcite.rel.RelDistribution;
-import org.apache.calcite.rel.RelDistribution.Type;
-import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.calcite.rel.RelFieldCollation;
+import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.io.AcidUtils.Operation;
@@ -30,6 +33,8 @@
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.opconventer.HiveOpConverter.OpAttr;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.util.DirectionUtils;
+import org.apache.hadoop.hive.ql.util.NullOrdering;
class HiveSortExchangeVisitor extends HiveRelNodeVisitor<HiveSortExchange> {
HiveSortExchangeVisitor(HiveOpConverter hiveOpConverter) {
@@ -50,25 +55,48 @@ OpAttr visit(HiveSortExchange exchangeRel) throws SemanticException {
}
RelDistribution distribution = exchangeRel.getDistribution();
- if (distribution.getType() != Type.HASH_DISTRIBUTED) {
- throw new SemanticException("Only hash distribution supported for LogicalExchange");
+ List<ExprNodeDesc> partitionKeyList;
+ switch (distribution.getType()) {
+ case HASH_DISTRIBUTED:
+ partitionKeyList = exchangeRel.getDistribution().getKeys().stream()
+ .map(keyIndex -> HiveOpConverterUtils.convertToExprNode(
+ exchangeRel.getCluster().getRexBuilder().makeInputRef(exchangeRel.getInput(), keyIndex),
+ exchangeRel.getInput(), inputOpAf.tabAlias, inputOpAf.vcolsInCalcite))
+ .collect(Collectors.toList());
+ break;
+
+ case ANY:
+ partitionKeyList = Collections.emptyList();
+ break;
+
+ default:
+ throw new SemanticException("Unsupported distribution type in HiveSortExchange: " + distribution.getType());
}
+
ExprNodeDesc[] expressions = new ExprNodeDesc[exchangeRel.getKeys().size()];
- for (int index = 0; index < exchangeRel.getKeys().size(); index++) {
+ StringBuilder order = new StringBuilder();
+ StringBuilder nullOrder = new StringBuilder();
+ for (int index = 0; index < exchangeRel.getCollation().getFieldCollations().size(); index++) {
+ RelFieldCollation fieldCollation = exchangeRel.getCollation().getFieldCollations().get(index);
expressions[index] = HiveOpConverterUtils.convertToExprNode(exchangeRel.getKeys().get(index),
exchangeRel.getInput(), inputOpAf.tabAlias, inputOpAf.vcolsInCalcite);
+
+ order.append(DirectionUtils.codeToSign(DirectionUtils.directionToCode(fieldCollation.getDirection())));
+ nullOrder.append(NullOrdering.fromDirection(fieldCollation.nullDirection).getSign());
}
exchangeRel.setKeyExpressions(expressions);
- ReduceSinkOperator rsOp = genReduceSink(inputOpAf.inputs.get(0), tabAlias, expressions,
- -1, -1, Operation.NOT_ACID, hiveOpConverter.getHiveConf());
+ Operator<?> inputOp = inputOpAf.inputs.get(0);
+ List<String> keepColumns = new ArrayList<>();
+ final List<ColumnInfo> inputSchema = inputOp.getSchema().getSignature();
+ for (ColumnInfo columnInfo : inputSchema) {
+ keepColumns.add(columnInfo.getInternalName());
+ }
- return new OpAttr(tabAlias, inputOpAf.vcolsInCalcite, rsOp);
- }
+ ReduceSinkOperator resultOp = HiveOpConverterUtils.genReduceSink(inputOpAf.inputs.get(0), tabAlias, expressions,
+ -1, partitionKeyList, order.toString(), nullOrder.toString(), -1, Operation.NOT_ACID,
+ hiveOpConverter.getHiveConf());
- private static ReduceSinkOperator genReduceSink(Operator<?> input, String tableAlias, ExprNodeDesc[] keys, int tag,
- int numReducers, Operation acidOperation, HiveConf hiveConf) throws SemanticException {
- return HiveOpConverterUtils.genReduceSink(input, tableAlias, keys, tag, new ArrayList<ExprNodeDesc>(), "", "",
- numReducers, acidOperation, hiveConf);
+ return new OpAttr(tabAlias, inputOpAf.vcolsInCalcite, resultOp);
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 73bcbe6304e..143f85769f3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -73,7 +73,6 @@
import org.apache.calcite.plan.volcano.RelSubset;
import org.apache.calcite.rel.AbstractRelNode;
import org.apache.calcite.rel.RelCollation;
-import org.apache.calcite.rel.RelCollationImpl;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelDistribution;
import org.apache.calcite.rel.RelDistributions;
@@ -379,6 +378,9 @@ public class CalcitePlanner extends SemanticAnalyzer {
* query with fully qualified identifiers.
*/
private static final String EXPANDED_QUERY_TOKEN_REWRITE_PROGRAM = "EXPANDED_QUERY_PROGRAM";
+ private static final String ERROR_MESSAGE_DUPLICATES_DETECTED =
+ "Duplicates detected when adding columns to RR: see previous message";
+
private final AtomicInteger noColsMissingStats = new AtomicInteger(0);
private SemanticException semanticException;
private boolean runCBO = true;
@@ -995,12 +997,6 @@ private static String canHandleQbForCbo(QueryProperties queryProperties,
HiveConf conf, boolean topLevelQB) {
List<String> reasons = new ArrayList<>();
// Not ok to run CBO, build error message.
- if (queryProperties.hasClusterBy()) {
- reasons.add("has cluster by");
- }
- if (queryProperties.hasDistributeBy()) {
- reasons.add("has distribute by");
- }
if (queryProperties.hasSortBy() && queryProperties.hasLimit()) {
reasons.add("has sort by with limit");
}
@@ -1858,7 +1854,8 @@ protected RelNode applyPreJoinOrderingTransforms(RelNode basePlan, RelMetadataPr
rules.add(HiveFilterTableFunctionTransposeRule.INSTANCE);
rules.add(HiveOptimizeInlineArrayTableFunctionRule.INSTANCE);
rules.add(HiveFilterSetOpTransposeRule.INSTANCE);
- rules.add(HiveFilterSortTransposeRule.INSTANCE);
+ rules.add(HiveFilterSortTransposeRule.SORT_LIMIT_INSTANCE);
+ rules.add(HiveFilterSortTransposeRule.SORT_EXCHANGE_INSTANCE);
rules.add(HiveFilterJoinRule.JOIN);
rules.add(HiveFilterJoinRule.FILTER_ON_JOIN);
rules.add(new HiveFilterAggregateTransposeRule(Filter.class, HiveRelFactories.HIVE_BUILDER,
@@ -2860,7 +2857,7 @@ join t1 using(a)
if (inputRels[0] != leftRel) {
RowResolver newLeftRR = new RowResolver();
if (!RowResolver.add(newLeftRR, leftRR)) {
- LOG.warn("Duplicates detected when adding columns to RR: see previous message");
+ LOG.warn(ERROR_MESSAGE_DUPLICATES_DETECTED);
}
for (int i = leftRel.getRowType().getFieldCount();
i < inputRels[0].getRowType().getFieldCount(); i++) {
@@ -2873,7 +2870,7 @@ join t1 using(a)
RowResolver joinRR = new RowResolver();
if (!RowResolver.add(joinRR, newLeftRR)) {
- LOG.warn("Duplicates detected when adding columns to RR: see previous message");
+ LOG.warn(ERROR_MESSAGE_DUPLICATES_DETECTED);
}
relToHiveColNameCalcitePosMap.put(topRel, buildHiveToCalciteColumnMap(joinRR));
relToHiveRR.put(topRel, joinRR);
@@ -2892,7 +2889,7 @@ join t1 using(a)
topRR = new RowResolver();
if (!RowResolver.add(topRR, leftRR)) {
- LOG.warn("Duplicates detected when adding columns to RR: see previous message");
+ LOG.warn(ERROR_MESSAGE_DUPLICATES_DETECTED);
}
} else {
final RelDataType combinedRowType = SqlValidatorUtil.createJoinType(
@@ -3928,26 +3925,18 @@ private RelNode genOBLogicalPlan(QB qb, Pair<RelNode, RowResolver> selPair,
}
}
- OBLogicalPlanGenState obLogicalPlanGenState = beginGenOBLogicalPlan(obAST, selPair, outermostOB);
-
- // 4. Construct SortRel
- RelTraitSet traitSet = cluster.traitSetOf(HiveRelNode.CONVENTION);
- RelCollation canonizedCollation = traitSet.canonize(
- RelCollationImpl.of(obLogicalPlanGenState.getFieldCollation()));
- RelNode sortRel;
+ RexNode offsetRN = null;
+ RexNode fetchRN = null;
if (limit != null) {
Integer offset = qb.getParseInfo().getDestLimitOffset(dest);
- RexNode offsetRN = (offset == null || offset == 0) ?
- null : cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(offset));
- RexNode fetchRN = cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(limit));
- sortRel = new HiveSortLimit(cluster, traitSet, obLogicalPlanGenState.getObInputRel(), canonizedCollation,
- offsetRN, fetchRN);
- } else {
- sortRel = new HiveSortLimit(cluster, traitSet, obLogicalPlanGenState.getObInputRel(), canonizedCollation,
- null, null);
+ offsetRN = (offset == null || offset == 0) ?
+ null : cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(offset));
+ fetchRN = cluster.getRexBuilder().makeExactLiteral(BigDecimal.valueOf(limit));
}
- return endGenOBLogicalPlan(obLogicalPlanGenState, sortRel);
+ return new OrderByRelBuilder(this, selPair, outermostOB)
+ .addSortByKeys(obAST)
+ .sortLimit(offsetRN, fetchRN);
}
private RelNode genSBLogicalPlan(QB qb, Pair<RelNode, RowResolver> selPair,
@@ -3955,203 +3944,31 @@ private RelNode genSBLogicalPlan(QB qb, Pair<RelNode, RowResolver> selPair,
QBParseInfo qbp = getQBParseInfo(qb);
String dest = qbp.getClauseNames().iterator().next();
ASTNode sbAST = qbp.getSortByForClause(dest);
+ ASTNode distributeByAST = qbp.getDistributeByForClause(dest);
- if (sbAST == null) {
+ if (sbAST == null && distributeByAST == null) {
return null;
}
- OBLogicalPlanGenState obLogicalPlanGenState = beginGenOBLogicalPlan(sbAST, selPair, outermostOB);
-
- // 4. Construct SortRel
- RelTraitSet traitSet = cluster.traitSetOf(HiveRelNode.CONVENTION);
- RelCollation canonizedCollation =
- traitSet.canonize(RelCollationImpl.of(obLogicalPlanGenState.getFieldCollation()));
- List<Integer> joinKeyPositions = new ArrayList<>(canonizedCollation.getFieldCollations().size());
- ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
- for (RelFieldCollation relFieldCollation : canonizedCollation.getFieldCollations()) {
- int index = relFieldCollation.getFieldIndex();
- joinKeyPositions.add(index);
- builder.add(cluster.getRexBuilder().makeInputRef(obLogicalPlanGenState.getObInputRel(), index));
- }
-
- RelNode sortRel = HiveSortExchange.create(
- obLogicalPlanGenState.getObInputRel(),
- // In case of SORT BY we do not need Distribution
- // but the instance RelDistributions.ANY can not be used here because
- // org.apache.calcite.rel.core.Exchange has
- // assert distribution != RelDistributions.ANY;
- new HiveRelDistribution(RelDistribution.Type.ANY, RelDistributions.ANY.getKeys()),
- canonizedCollation,
- builder.build());
-
- return endGenOBLogicalPlan(obLogicalPlanGenState, sortRel);
+ return new OrderByRelBuilder(this, selPair, outermostOB)
+ .addSortByKeys(sbAST)
+ .addRelDistribution(distributeByAST)
+ .sortExchange();
}
- // - Walk through OB exprs and extract field collations and additional virtual columns needed
- // - Add Child Project Rel if needed,
- // - Generate Output RR, input Sel Rel for top constraining Sel
- private OBLogicalPlanGenState beginGenOBLogicalPlan(
- ASTNode obAST, Pair<RelNode, RowResolver> selPair, boolean outermostOB) throws SemanticException {
- // selPair.getKey() is the operator right before OB
- // selPair.getValue() is RR which only contains columns needed in result
- // set. Extra columns needed by order by will be absent from it.
- RelNode srcRel = selPair.getKey();
- RowResolver selectOutputRR = selPair.getValue();
-
- // 2. Walk through OB exprs and extract field collations and additional
- // virtual columns needed
- final List<RexNode> newVCLst = new ArrayList<>();
- final List<RelFieldCollation> fieldCollations = Lists.newArrayList();
- int fieldIndex = 0;
-
- List<Node> obASTExprLst = obAST.getChildren();
- List<Pair<ASTNode, TypeInfo>> vcASTTypePairs = new ArrayList<>();
- RowResolver inputRR = relToHiveRR.get(srcRel);
- RowResolver outputRR = new RowResolver();
-
- int srcRelRecordSz = srcRel.getRowType().getFieldCount();
-
- for (int i = 0; i < obASTExprLst.size(); i++) {
- // 2.1 Convert AST Expr to ExprNode
- ASTNode orderByNode = (ASTNode) obASTExprLst.get(i);
- ASTNode nullObASTExpr = (ASTNode) orderByNode.getChild(0);
- ASTNode ref = (ASTNode) nullObASTExpr.getChild(0);
-
- boolean isBothByPos = HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS);
- boolean isObyByPos = isBothByPos
- || HiveConf.getBoolVar(conf, ConfVars.HIVE_ORDERBY_POSITION_ALIAS);
- // replace each of the position alias in ORDERBY with the actual column
- if (ref != null && ref.getToken().getType() == HiveParser.Number) {
- if (isObyByPos) {
- fieldIndex = getFieldIndexFromColumnNumber(selectOutputRR, ref);
- } else { // if not using position alias and it is a number.
- LOG.warn("Using constant number "
- + ref.getText()
- + " in order by. If you try to use position alias when hive.orderby.position.alias is false, " +
- "the position alias will be ignored.");
- }
- } else {
- // 2.2 Convert ExprNode to RexNode
- RexNode orderByExpression = getOrderByExpression(selectOutputRR, inputRR, orderByNode, ref);
-
- // 2.3 Determine the index of ob expr in child schema
- // NOTE: Calcite can not take compound exprs in OB without it being
- // present in the child (& hence we add a child Project Rel)
- if (orderByExpression instanceof RexInputRef) {
- fieldIndex = ((RexInputRef) orderByExpression).getIndex();
- } else {
- fieldIndex = srcRelRecordSz + newVCLst.size();
- newVCLst.add(orderByExpression);
- vcASTTypePairs.add(new Pair<>(ref, TypeConverter.convert(orderByExpression.getType())));
- }
- }
-
- // 2.4 Determine the Direction of order by
- RelFieldCollation.Direction order = RelFieldCollation.Direction.DESCENDING;
- if (orderByNode.getType() == HiveParser.TOK_TABSORTCOLNAMEASC) {
- order = RelFieldCollation.Direction.ASCENDING;
- }
- RelFieldCollation.NullDirection nullOrder;
- if (nullObASTExpr.getType() == HiveParser.TOK_NULLS_FIRST) {
- nullOrder = RelFieldCollation.NullDirection.FIRST;
- } else if (nullObASTExpr.getType() == HiveParser.TOK_NULLS_LAST) {
- nullOrder = RelFieldCollation.NullDirection.LAST;
- } else {
- throw new SemanticException("Unexpected null ordering option: "
- + nullObASTExpr.getType());
- }
-
- // 2.5 Add to field collations
- fieldCollations.add(new RelFieldCollation(fieldIndex, order, nullOrder));
- }
-
- // 3. Add Child Project Rel if needed, Generate Output RR, input Sel Rel
- // for top constraining Sel
- RelNode obInputRel = srcRel;
- if (!newVCLst.isEmpty()) {
- List<RexNode> originalInputRefs = toRexNodeList(srcRel);
- RowResolver obSyntheticProjectRR = new RowResolver();
- if (!RowResolver.add(obSyntheticProjectRR, inputRR)) {
- throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message",
- UnsupportedFeature.Duplicates_in_RR);
- }
- int vcolPos = inputRR.getRowSchema().getSignature().size();
- for (Pair<ASTNode, TypeInfo> astTypePair : vcASTTypePairs) {
- obSyntheticProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo(
- SemanticAnalyzer.getColumnInternalName(vcolPos), astTypePair.getValue(), null,
- false));
- vcolPos++;
- }
- obInputRel = genSelectRelNode(CompositeList.of(originalInputRefs, newVCLst),
- obSyntheticProjectRR, srcRel);
-
- if (outermostOB) {
- if (!RowResolver.add(outputRR, inputRR)) {
- throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message",
- UnsupportedFeature.Duplicates_in_RR);
- }
-
- } else {
- if (!RowResolver.add(outputRR, obSyntheticProjectRR)) {
- throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message",
- UnsupportedFeature.Duplicates_in_RR);
- }
- }
- } else {
- if (!RowResolver.add(outputRR, inputRR)) {
- throw new CalciteSemanticException(
- "Duplicates detected when adding columns to RR: see previous message",
- UnsupportedFeature.Duplicates_in_RR);
- }
- }
- return new OBLogicalPlanGenState(obInputRel, fieldCollations, selectOutputRR, outputRR, srcRel);
- }
+ private RelNode genClusterByLogicalPlan(QB qb, Pair<RelNode, RowResolver> selPair,
+ boolean outermostOB) throws SemanticException {
+ QBParseInfo qbp = getQBParseInfo(qb);
+ String dest = qbp.getClauseNames().iterator().next();
+ ASTNode clusterByAST = qbp.getClusterByForClause(dest);
- private RexNode getOrderByExpression(
- RowResolver selectOutputRR, RowResolver inputRR, ASTNode orderByNode, ASTNode ref)
- throws SemanticException {
- // first try to get it from select
- // in case of udtf, selectOutputRR may be null.
- RexNode orderByExpression = null;
- if (selectOutputRR != null) {
- try {
- Map<ASTNode, RexNode> astToExprNDescMap = genAllRexNode(ref, selectOutputRR, cluster.getRexBuilder());
- orderByExpression = astToExprNDescMap.get(ref);
- } catch (SemanticException ex) {
- // we can tolerate this as this is the previous behavior
- LOG.debug("Can not find column in " + ref.getText() + ". The error msg is "
- + ex.getMessage());
- }
- }
- // then try to get it from all
- if (orderByExpression == null) {
- Map<ASTNode, RexNode> astToExprNDescMap = genAllRexNode(ref, inputRR, cluster.getRexBuilder());
- orderByExpression = astToExprNDescMap.get(ref);
- }
- if (orderByExpression == null) {
- throw new SemanticException("Invalid order by expression: " + orderByNode.toString());
+ if (clusterByAST == null) {
+ return null;
}
- return orderByExpression;
- }
- // SELECT a, b FROM t ORDER BY 1
- private int getFieldIndexFromColumnNumber(RowResolver selectOutputRR, ASTNode ref) throws SemanticException {
- int fieldIndex;
- int pos = Integer.parseInt(ref.getText());
- if (pos > 0 && pos <= selectOutputRR.getColumnInfos().size()) {
- // fieldIndex becomes so simple
- // Note that pos starts from 1 while fieldIndex starts from 0;
- fieldIndex = pos - 1;
- } else {
- throw new SemanticException(
- ErrorMsg.INVALID_POSITION_ALIAS_IN_ORDERBY.getMsg("Position alias: " + pos
- + " does not exist\n" + "The Select List is indexed from 1 to "
- + selectOutputRR.getColumnInfos().size()));
- }
- return fieldIndex;
+ return new OrderByRelBuilder(this, selPair, outermostOB)
+ .addClusterBy(clusterByAST)
+ .sortExchange();
}
private List<RexNode> toRexNodeList(RelNode srcRel) {
@@ -4160,31 +3977,6 @@ private List<RexNode> toRexNodeList(RelNode srcRel) {
.collect(Collectors.toList());
}
- // 5. Update RR maps
- // NOTE: Output RR for SortRel is considered same as its input; we may
- // end up not using VC that is present in sort rel. Also note that
- // rowtype of sortrel is the type of it child; if child happens to be
- // synthetic project that we introduced then that projectrel would
- // contain the vc.
- public RelNode endGenOBLogicalPlan(OBLogicalPlanGenState obLogicalPlanGenState, RelNode sortRel)
- throws CalciteSemanticException {
-
- ImmutableMap<String, Integer> hiveColNameCalcitePosMap =
- buildHiveToCalciteColumnMap(obLogicalPlanGenState.getOutputRR());
- relToHiveRR.put(sortRel, obLogicalPlanGenState.getOutputRR());
- relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap);
-
- if (obLogicalPlanGenState.getSelectOutputRR() != null) {
- List<RexNode> originalInputRefs = toRexNodeList(obLogicalPlanGenState.getSrcRel());
- List<RexNode> selectedRefs = originalInputRefs.subList(
- 0, obLogicalPlanGenState.getSelectOutputRR().getColumnInfos().size());
- // We need to add select since order by schema may have more columns than result schema.
- return genSelectRelNode(selectedRefs, obLogicalPlanGenState.getSelectOutputRR(), sortRel);
- } else {
- return sortRel;
- }
- }
-
private RelNode genLimitLogicalPlan(QB qb, RelNode srcRel) throws SemanticException {
HiveRelNode sortRel = null;
QBParseInfo qbp = getQBParseInfo(qb);
@@ -4388,7 +4180,7 @@ private RelNode genSelectForWindowing(QB qb, RelNode srcRel, HashSet<ColumnInfo>
// 3. Construct new Row Resolver with everything from below.
RowResolver out_rwsch = new RowResolver();
if (!RowResolver.add(out_rwsch, inputRR)) {
- LOG.warn("Duplicates detected when adding columns to RR: see previous message");
+ LOG.warn(ERROR_MESSAGE_DUPLICATES_DETECTED);
}
// 4. Walk through Window Expressions & Construct RexNodes for those,
@@ -5198,7 +4990,12 @@ private RelNode genLogicalPlan(QB qb, boolean outerMostQB,
} else {
// 7. Build Rel for Sort By Clause
sbRel = genSBLogicalPlan(qb, selPair, outerMostQB);
- srcRel = (sbRel == null) ? srcRel : sbRel;
+ if (sbRel != null) {
+ srcRel = sbRel;
+ } else {
+ sbRel = genClusterByLogicalPlan(qb, selPair, outerMostQB);
+ srcRel = (sbRel == null) ? srcRel : sbRel;
+ }
// 8. Build Rel for Limit Clause
limitRel = genLimitLogicalPlan(qb, srcRel);
@@ -5313,47 +5110,6 @@ private QBParseInfo getQBParseInfo(QB qb) throws CalciteSemanticException {
}
}
- /**
- * This class stores the partial results of Order/Sort by clause logical plan generation.
- * See {@link CalcitePlannerAction#beginGenOBLogicalPlan}, {@link CalcitePlannerAction#endGenOBLogicalPlan}
- */
- private static class OBLogicalPlanGenState {
- private final RelNode obInputRel;
- private final List<RelFieldCollation> canonizedCollation;
- private final RowResolver selectOutputRR;
- private final RowResolver outputRR;
- private final RelNode srcRel;
-
- OBLogicalPlanGenState(RelNode obInputRel, List<RelFieldCollation> canonizedCollation,
- RowResolver selectOutputRR, RowResolver outputRR, RelNode srcRel) {
- this.obInputRel = obInputRel;
- this.canonizedCollation = canonizedCollation;
- this.selectOutputRR = selectOutputRR;
- this.outputRR = outputRR;
- this.srcRel = srcRel;
- }
-
- public RelNode getObInputRel() {
- return obInputRel;
- }
-
- public List<RelFieldCollation> getFieldCollation() {
- return canonizedCollation;
- }
-
- public RowResolver getSelectOutputRR() {
- return selectOutputRR;
- }
-
- public RowResolver getOutputRR() {
- return outputRR;
- }
-
- public RelNode getSrcRel() {
- return srcRel;
- }
- }
-
@Override
protected Table getTableObjectByName(String tabName, boolean throwException) throws HiveException {
String[] names = Utilities.getDbTableName(tabName);
@@ -5611,4 +5367,280 @@ private enum TableType {
JDBC
}
+ private class OrderByRelBuilder {
+ private final CalcitePlannerAction calcitePlannerAction;
+ private final List<RexNode> newVCLst = new ArrayList<>();
+ private final List<RelFieldCollation> fieldCollations = Lists.newArrayList();
+ private final List<Pair<ASTNode, TypeInfo>> vcASTTypePairs = new ArrayList<>();
+ private final RowResolver outputRR = new RowResolver();
+ private final Pair<RelNode, RowResolver> selPair;
+ private final boolean outermostOB;
+
+ private HiveRelDistribution hiveRelDistribution;
+ private RelNode obInputRel;
+
+ OrderByRelBuilder(
+ CalcitePlannerAction calcitePlannerAction, Pair<RelNode, RowResolver> selPair, boolean outermostOB) {
+ this.calcitePlannerAction = calcitePlannerAction;
+ this.selPair = selPair;
+ this.outermostOB = outermostOB;
+ }
+
+ // - Walk through OB exprs and extract field collations and additional virtual columns needed
+ // - Add Child Project Rel if needed,
+ // - Generate Output RR, input Sel Rel for top constraining Sel
+ OrderByRelBuilder addSortByKeys(ASTNode obAST) throws SemanticException {
+ if (obAST == null) {
+ return this;
+ }
+
+ // 2. Walk through OB exprs and extract field collations and additional
+ // virtual columns needed
+ List<Node> obASTExprLst = obAST.getChildren();
+
+ for (int i = 0; i < obASTExprLst.size(); i++) {
+ // 2.1 Convert AST Expr to ExprNode
+ ASTNode orderByNode = (ASTNode) obASTExprLst.get(i);
+ ASTNode nullObASTExpr = (ASTNode) orderByNode.getChild(0);
+ ASTNode ref = (ASTNode) nullObASTExpr.getChild(0);
+
+ int fieldIndex = genSortByKey(ref);
+
+ // 2.4 Determine the Direction of order by
+ RelFieldCollation.Direction order = RelFieldCollation.Direction.DESCENDING;
+ if (orderByNode.getType() == HiveParser.TOK_TABSORTCOLNAMEASC) {
+ order = RelFieldCollation.Direction.ASCENDING;
+ }
+ RelFieldCollation.NullDirection nullOrder;
+ if (nullObASTExpr.getType() == HiveParser.TOK_NULLS_FIRST) {
+ nullOrder = RelFieldCollation.NullDirection.FIRST;
+ } else if (nullObASTExpr.getType() == HiveParser.TOK_NULLS_LAST) {
+ nullOrder = RelFieldCollation.NullDirection.LAST;
+ } else {
+ throw new SemanticException("Unexpected null ordering option: "
+ + nullObASTExpr.getType());
+ }
+
+ // 2.5 Add to field collations
+ fieldCollations.add(new RelFieldCollation(fieldIndex, order, nullOrder));
+ }
+
+ return this;
+ }
+
+ private int genSortByKey(ASTNode ref) throws SemanticException {
+ // selPair.getKey() is the operator right before OB
+ // selPair.getValue() is RR which only contains columns needed in result
+ // set. Extra columns needed by order by will be absent from it.
+ RelNode srcRel = selPair.getKey();
+ RowResolver selectOutputRR = selPair.getValue();
+ RowResolver inputRR = calcitePlannerAction.relToHiveRR.get(srcRel);
+
+ int srcRelRecordSz = srcRel.getRowType().getFieldCount();
+ boolean isBothByPos = HiveConf.getBoolVar(conf, ConfVars.HIVE_GROUPBY_ORDERBY_POSITION_ALIAS);
+ boolean isObyByPos = isBothByPos
+ || HiveConf.getBoolVar(conf, ConfVars.HIVE_ORDERBY_POSITION_ALIAS);
+ // replace each of the position alias in ORDERBY with the actual column
+ if (ref != null && ref.getToken().getType() == HiveParser.Number) {
+ if (isObyByPos) {
+ return getFieldIndexFromColumnNumber(selectOutputRR, ref);
+ } else { // if not using position alias and it is a number.
+ LOG.warn("Using constant number {}" +
+ " in order by. If you try to use position alias when hive.orderby.position.alias is false, " +
+ "the position alias will be ignored.", ref.getText());
+ }
+ } else {
+ // 2.2 Convert ExprNode to RexNode
+ RexNode orderByExpression = getOrderByExpression(selectOutputRR, inputRR, ref);
+
+ // 2.3 Determine the index of ob expr in child schema
+ // NOTE: Calcite can not take compound exprs in OB without it being
+ // present in the child (& hence we add a child Project Rel)
+ if (orderByExpression instanceof RexInputRef) {
+ return ((RexInputRef) orderByExpression).getIndex();
+ } else {
+ int fieldIndex = srcRelRecordSz + newVCLst.size();
+ newVCLst.add(orderByExpression);
+ vcASTTypePairs.add(new Pair<>(ref, TypeConverter.convert(orderByExpression.getType())));
+ return fieldIndex;
+ }
+ }
+
+ return 0;
+ }
+
+ private RexNode getOrderByExpression(
+ RowResolver selectOutputRR, RowResolver inputRR, ASTNode ref)
+ throws SemanticException {
+ // first try to get it from select
+ // in case of udtf, selectOutputRR may be null.
+ RexNode orderByExpression = null;
+ if (selectOutputRR != null) {
+ try {
+ Map<ASTNode, RexNode> astToExprNDescMap = genAllRexNode(
+ ref, selectOutputRR, calcitePlannerAction.cluster.getRexBuilder());
+ orderByExpression = astToExprNDescMap.get(ref);
+ } catch (SemanticException ex) {
+ // we can tolerate this as this is the previous behavior
+ LOG.debug("Can not find column in {} The error msg is {}", ref.getText(), ex.getMessage());
+ }
+ }
+ // then try to get it from all
+ if (orderByExpression == null) {
+ Map<ASTNode, RexNode> astToExprNDescMap = genAllRexNode(
+ ref, inputRR, calcitePlannerAction.cluster.getRexBuilder());
+ orderByExpression = astToExprNDescMap.get(ref);
+ }
+ if (orderByExpression == null) {
+ throw new SemanticException("Invalid order by expression: " + ref.toString());
+ }
+ return orderByExpression;
+ }
+
+ // SELECT a, b FROM t ORDER BY 1
+ private int getFieldIndexFromColumnNumber(RowResolver selectOutputRR, ASTNode ref) throws SemanticException {
+ int fieldIndex;
+ int pos = Integer.parseInt(ref.getText());
+ if (pos > 0 && pos <= selectOutputRR.getColumnInfos().size()) {
+ // fieldIndex becomes so simple
+ // Note that pos starts from 1 while fieldIndex starts from 0;
+ fieldIndex = pos - 1;
+ } else {
+ throw new SemanticException(
+ ErrorMsg.INVALID_POSITION_ALIAS_IN_ORDERBY.getMsg("Position alias: " + pos
+ + " does not exist\n" + "The Select List is indexed from 1 to "
+ + selectOutputRR.getColumnInfos().size()));
+ }
+ return fieldIndex;
+ }
+
+ OrderByRelBuilder addRelDistribution(ASTNode distributeByAST) throws SemanticException {
+ if (distributeByAST != null) {
+ Builder<Integer> keys = ImmutableList.builder();
+ for (int i = 0; i < distributeByAST.getChildCount(); ++i) {
+ ASTNode keyAST = (ASTNode) distributeByAST.getChild(i);
+ int fieldIndex = genSortByKey(keyAST);
+ keys.add(fieldIndex);
+ }
+ hiveRelDistribution = new HiveRelDistribution(RelDistribution.Type.HASH_DISTRIBUTED, keys.build());
+ } else {
+ // In case of SORT BY we do not need Distribution
+ // but the instance RelDistributions.ANY can not be used here because
+ // org.apache.calcite.rel.core.Exchange has
+ // assert distribution != RelDistributions.ANY;
+ hiveRelDistribution = new HiveRelDistribution(RelDistribution.Type.ANY, RelDistributions.ANY.getKeys());
+ }
+ return this;
+ }
+
+ OrderByRelBuilder addClusterBy(ASTNode clusterBy) throws SemanticException {
+ addRelDistribution(clusterBy);
+ for (Integer fieldIndex : hiveRelDistribution.getKeys()) {
+ fieldCollations.add(new RelFieldCollation(
+ fieldIndex, RelFieldCollation.Direction.ASCENDING, RelFieldCollation.NullDirection.FIRST));
+ }
+ return this;
+ }
+
+ // - Walk through OB exprs and extract field collations and additional virtual columns needed
+ // - Add Child Project Rel if needed,
+ // - Generate Output RR, input Sel Rel for top constraining Sel
+ private void genOBProject() throws SemanticException {
+ // selPair.getKey() is the operator right before OB
+ // selPair.getValue() is RR which only contains columns needed in result
+ // set. Extra columns needed by order by will be absent from it.
+ RelNode srcRel = selPair.getKey();
+ RowResolver inputRR = calcitePlannerAction.relToHiveRR.get(srcRel);
+
+ // 3. Add Child Project Rel if needed, Generate Output RR, input Sel Rel
+ // for top constraining Sel
+ obInputRel = srcRel;
+ if (!newVCLst.isEmpty()) {
+ List<RexNode> originalInputRefs = calcitePlannerAction.toRexNodeList(srcRel);
+ RowResolver obSyntheticProjectRR = new RowResolver();
+ if (!RowResolver.add(obSyntheticProjectRR, inputRR)) {
+ throw new CalciteSemanticException(
+ ERROR_MESSAGE_DUPLICATES_DETECTED,
+ UnsupportedFeature.Duplicates_in_RR);
+ }
+ int vcolPos = inputRR.getRowSchema().getSignature().size();
+ for (Pair<ASTNode, TypeInfo> astTypePair : vcASTTypePairs) {
+ obSyntheticProjectRR.putExpression(astTypePair.getKey(), new ColumnInfo(
+ SemanticAnalyzer.getColumnInternalName(vcolPos), astTypePair.getValue(), null,
+ false));
+ vcolPos++;
+ }
+ obInputRel = calcitePlannerAction.genSelectRelNode(CompositeList.of(originalInputRefs, newVCLst),
+ obSyntheticProjectRR, srcRel);
+
+ if (outermostOB) {
+ if (!RowResolver.add(outputRR, inputRR)) {
+ throw new CalciteSemanticException(
+ ERROR_MESSAGE_DUPLICATES_DETECTED,
+ UnsupportedFeature.Duplicates_in_RR);
+ }
+
+ } else {
+ if (!RowResolver.add(outputRR, obSyntheticProjectRR)) {
+ throw new CalciteSemanticException(
+ ERROR_MESSAGE_DUPLICATES_DETECTED,
+ UnsupportedFeature.Duplicates_in_RR);
+ }
+ }
+ } else {
+ if (!RowResolver.add(outputRR, inputRR)) {
+ throw new CalciteSemanticException(
+ ERROR_MESSAGE_DUPLICATES_DETECTED,
+ UnsupportedFeature.Duplicates_in_RR);
+ }
+ }
+ }
+
+ RelNode sortLimit(RexNode offsetRN, RexNode fetchRN) throws SemanticException {
+ genOBProject();
+
+ // 4. Construct SortRel
+ RelOptCluster cluster = calcitePlannerAction.cluster;
+ RelTraitSet traitSet = cluster.traitSetOf(HiveRelNode.CONVENTION);
+ RelCollation canonizedCollation = RelCollations.of(fieldCollations);
+ RelNode sortRel = new HiveSortLimit(cluster, traitSet, obInputRel, canonizedCollation, offsetRN, fetchRN);
+ return endGenOBLogicalPlan(sortRel);
+ }
+
+ RelNode sortExchange() throws SemanticException {
+ genOBProject();
+
+ RelCollation canonizedCollation = RelCollations.of(fieldCollations);
+ ImmutableList.Builder<RexNode> builder = ImmutableList.builder();
+ for (RelFieldCollation relFieldCollation : canonizedCollation.getFieldCollations()) {
+ int index = relFieldCollation.getFieldIndex();
+ builder.add(calcitePlannerAction.cluster.getRexBuilder().makeInputRef(obInputRel, index));
+ }
+ RelNode sortRel = HiveSortExchange.create(obInputRel, hiveRelDistribution, canonizedCollation, builder.build());
+ return endGenOBLogicalPlan(sortRel);
+ }
+
+ // 5. Update RR maps
+ // NOTE: Output RR for SortRel is considered same as its input; we may
+ // end up not using VC that is present in sort rel. Also note that
+ // rowtype of sortrel is the type of it child; if child happens to be
+ // synthetic project that we introduced then that projectrel would
+ // contain the vc.
+ private RelNode endGenOBLogicalPlan(RelNode sortRel) throws CalciteSemanticException {
+ ImmutableMap<String, Integer> hiveColNameCalcitePosMap =
+ calcitePlannerAction.buildHiveToCalciteColumnMap(outputRR);
+ calcitePlannerAction.relToHiveRR.put(sortRel, outputRR);
+ calcitePlannerAction.relToHiveColNameCalcitePosMap.put(sortRel, hiveColNameCalcitePosMap);
+
+ RowResolver selectOutputRR = selPair.getValue();
+ if (selectOutputRR != null) {
+ List<RexNode> originalInputRefs = calcitePlannerAction.toRexNodeList(selPair.getKey());
+ List<RexNode> selectedRefs = originalInputRefs.subList(0, selectOutputRR.getColumnInfos().size());
+ // We need to add select since order by schema may have more columns than result schema.
+ return calcitePlannerAction.genSelectRelNode(selectedRefs, selectOutputRR, sortRel);
+ }
+
+ return sortRel;
+ }
+ }
}
diff --git a/ql/src/test/queries/clientnegative/qualify_no_cbo_2.q b/ql/src/test/queries/clientnegative/qualify_no_cbo_2.q
deleted file mode 100644
index a95a727c479..00000000000
--- a/ql/src/test/queries/clientnegative/qualify_no_cbo_2.q
+++ /dev/null
@@ -1,8 +0,0 @@
-create table t1 (a int, b string, c int);
-
--- distribute by is not supported by cbo so this statement should fall back to non cbo path and fail
--- since qualify requires CBO
-select a, b, c, row_number() over (partition by b order by c) as row_num
- from t1
-qualify row_num = 1
-distribute by a;
diff --git a/ql/src/test/queries/clientpositive/cbo_distribute_sort_cluster_by.q b/ql/src/test/queries/clientpositive/cbo_distribute_sort_cluster_by.q
new file mode 100644
index 00000000000..aa8c7d92372
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_distribute_sort_cluster_by.q
@@ -0,0 +1,21 @@
+create table t1 (a string, b int);
+
+-- distribute by
+explain cbo
+select * from t1 distribute by a;
+explain
+select * from t1 distribute by a;
+
+-- distribute by and sort by
+explain cbo
+select * from t1 distribute by a, b sort by a;
+
+explain
+select * from t1 distribute by a, b sort by a;
+
+-- cluster by
+explain cbo
+select * from t1 cluster by a, b;
+
+explain
+select * from t1 cluster by a, b;
diff --git a/ql/src/test/queries/clientpositive/cbo_rp_distribute_sort_cluster_by.q b/ql/src/test/queries/clientpositive/cbo_rp_distribute_sort_cluster_by.q
new file mode 100644
index 00000000000..1677aadf20d
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/cbo_rp_distribute_sort_cluster_by.q
@@ -0,0 +1,15 @@
+set hive.cbo.returnpath.hiveop=true;
+
+create table t1 (a string, b int);
+
+-- distribute by
+explain
+select * from t1 where a between date_add('2014-03-14', -1) and '2014-03-14' distribute by a;
+
+-- distribute by and sort by
+explain
+select * from t1 distribute by a, b sort by a;
+
+-- cluster by
+explain
+select * from t1 cluster by a, b;
diff --git a/ql/src/test/results/clientnegative/qualify_no_cbo_2.q.out b/ql/src/test/results/clientnegative/qualify_no_cbo_2.q.out
deleted file mode 100644
index 97f2f2d17e1..00000000000
--- a/ql/src/test/results/clientnegative/qualify_no_cbo_2.q.out
+++ /dev/null
@@ -1,9 +0,0 @@
-PREHOOK: query: create table t1 (a int, b string, c int)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@t1
-POSTHOOK: query: create table t1 (a int, b string, c int)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@t1
-FAILED: SemanticException [Error 10433]: The following functionality requires CBO (hive.cbo.enable): Qualify clause
diff --git a/ql/src/test/results/clientpositive/llap/cbo_distribute_sort_cluster_by.q.out b/ql/src/test/results/clientpositive/llap/cbo_distribute_sort_cluster_by.q.out
new file mode 100644
index 00000000000..60e22ff8a76
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/cbo_distribute_sort_cluster_by.q.out
@@ -0,0 +1,234 @@
+PREHOOK: query: create table t1 (a string, b int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1 (a string, b int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: explain cbo
+select * from t1 distribute by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+select * from t1 distribute by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+CBO PLAN:
+HiveSortExchange(distribution=[hash[0]], collation=[[]])
+ HiveProject(a=[$0], b=[$1])
+ HiveTableScan(table=[[default, t1]], table:alias=[t1])
+
+PREHOOK: query: explain
+select * from t1 distribute by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t1 distribute by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string), b (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain cbo
+select * from t1 distribute by a, b sort by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+select * from t1 distribute by a, b sort by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+CBO PLAN:
+HiveSortExchange(distribution=[hash[0, 1]], collation=[[0]])
+ HiveProject(a=[$0], b=[$1])
+ HiveTableScan(table=[[default, t1]], table:alias=[t1])
+
+PREHOOK: query: explain
+select * from t1 distribute by a, b sort by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t1 distribute by a, b sort by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string), b (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain cbo
+select * from t1 cluster by a, b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain cbo
+select * from t1 cluster by a, b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+CBO PLAN:
+HiveSortExchange(distribution=[hash[0, 1]], collation=[[0 ASC-nulls-first, 1 ASC-nulls-first]])
+ HiveProject(a=[$0], b=[$1])
+ HiveTableScan(table=[[default, t1]], table:alias=[t1])
+
+PREHOOK: query: explain
+select * from t1 cluster by a, b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t1 cluster by a, b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string), b (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: int)
+ null sort order: aa
+ sort order: ++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: int)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
diff --git a/ql/src/test/results/clientpositive/llap/cbo_rp_distribute_sort_cluster_by.q.out b/ql/src/test/results/clientpositive/llap/cbo_rp_distribute_sort_cluster_by.q.out
new file mode 100644
index 00000000000..be4cc749290
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/cbo_rp_distribute_sort_cluster_by.q.out
@@ -0,0 +1,181 @@
+PREHOOK: query: create table t1 (a string, b int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@t1
+POSTHOOK: query: create table t1 (a string, b int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@t1
+PREHOOK: query: explain
+select * from t1 where a between date_add('2014-03-14', -1) and '2014-03-14' distribute by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t1 where a between date_add('2014-03-14', -1) and '2014-03-14' distribute by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (CUSTOM_SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ filterExpr: CAST( a AS DATE) BETWEEN DATE'2014-03-13' AND DATE'2014-03-14' (type: boolean)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: CAST( a AS DATE) BETWEEN DATE'2014-03-13' AND DATE'2014-03-14' (type: boolean)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string), b (type: int)
+ outputColumnNames: a, b
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ null sort order:
+ sort order:
+ Map-reduce partition columns: a (type: string)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ value expressions: a (type: string), b (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select * from t1 distribute by a, b sort by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t1 distribute by a, b sort by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string), b (type: int)
+ outputColumnNames: a, b
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: a (type: string), b (type: int)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ value expressions: b (type: int)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain
+select * from t1 cluster by a, b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: explain
+select * from t1 cluster by a, b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: string), b (type: int)
+ outputColumnNames: a, b
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: string), b (type: int)
+ null sort order: aa
+ sort order: ++
+ Map-reduce partition columns: a (type: string), b (type: int)
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Operator Tree:
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 188 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
diff --git a/ql/src/test/results/clientpositive/llap/cluster.q.out b/ql/src/test/results/clientpositive/llap/cluster.q.out
index b4c36f3e415..ecdade88f50 100644
--- a/ql/src/test/results/clientpositive/llap/cluster.q.out
+++ b/ql/src/test/results/clientpositive/llap/cluster.q.out
@@ -24,21 +24,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 10) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 10.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 10) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 10.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -48,10 +48,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -98,21 +98,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -122,10 +122,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -172,21 +172,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -196,10 +196,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -246,21 +246,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -270,10 +270,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -320,21 +320,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -344,10 +344,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -394,21 +394,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -418,10 +418,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -468,21 +468,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col1 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col0 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -492,10 +492,10 @@ STAGE PLANS:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -542,21 +542,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -566,10 +566,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -617,35 +617,43 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -655,31 +663,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col2 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col2 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -727,36 +731,44 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -766,31 +778,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6, _col7
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string), _col7 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col2 (type: string), _col3 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -838,36 +846,44 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -877,31 +893,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6, _col7
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string), _col7 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col2 (type: string), _col3 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string), VALUE._col2 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 712 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -949,35 +961,43 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -987,31 +1007,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: string), _col2 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string), _col2 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -1072,10 +1088,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src
- filterExpr: (key < 100) (type: boolean)
+ filterExpr: (UDFToDouble(key) < 100.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key < 100) (type: boolean)
+ predicate: (UDFToDouble(key) < 100.0D) (type: boolean)
Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
@@ -1094,10 +1110,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src
- filterExpr: (key > 100) (type: boolean)
+ filterExpr: (UDFToDouble(key) > 100.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key > 100) (type: boolean)
+ predicate: (UDFToDouble(key) > 100.0D) (type: boolean)
Statistics: Num rows: 166 Data size: 29548 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
diff --git a/ql/src/test/results/clientpositive/llap/columnarserde_create_shortcut.q.out b/ql/src/test/results/clientpositive/llap/columnarserde_create_shortcut.q.out
index 4dc1368a841..9b0a95bf1d7 100644
--- a/ql/src/test/results/clientpositive/llap/columnarserde_create_shortcut.q.out
+++ b/ql/src/test/results/clientpositive/llap/columnarserde_create_shortcut.q.out
@@ -44,7 +44,7 @@ STAGE PLANS:
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: 1 (type: int)
+ Map-reduce partition columns: _col0 (type: array<int>)
Statistics: Num rows: 11 Data size: 34628 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<int>), _col1 (type: array<string>), _col2 (type: map<string,string>), _col3 (type: int), _col4 (type: string)
Execution mode: vectorized, llap
diff --git a/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out b/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out
index 1a61b67f72e..a571b073be9 100644
--- a/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out
+++ b/ql/src/test/results/clientpositive/llap/correlationoptimizer14.q.out
@@ -167,34 +167,42 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
+ filterExpr: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate: key is not null (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: z
- sort order: +
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 3
Map Operator Tree:
TableScan
alias: y
+ filterExpr: key is not null (type: boolean)
Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
+ Filter Operator
+ predicate: key is not null (type: boolean)
Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: z
- sort order: +
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -203,34 +211,28 @@ STAGE PLANS:
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 25 Data size: 4375 Basic stats: COMPLETE Column stats: COMPLETE
- Dummy Store
+ Dummy Store
Execution mode: llap
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: _col0 is not null (type: boolean)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Merge Join Operator
- condition map:
- Inner Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 39 Data size: 13767 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
Statistics: Num rows: 39 Data size: 13767 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 39 Data size: 13767 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
diff --git a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
index b9cf01811e1..6d1bc982c8f 100644
--- a/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
+++ b/ql/src/test/results/clientpositive/llap/dynpart_sort_optimization2.q.out
@@ -102,10 +102,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss_n0
- filterExpr: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ filterExpr: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ predicate: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: ss_sold_date_sk (type: int), ss_net_paid_inc_tax (type: float), ss_net_profit (type: float)
@@ -378,10 +378,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss_n0
- filterExpr: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ filterExpr: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ predicate: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ss_net_paid_inc_tax (type: float), ss_net_profit (type: float), ss_sold_date_sk (type: int)
@@ -648,10 +648,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss_n0
- filterExpr: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ filterExpr: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ predicate: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Group By Operator
keys: ss_sold_date_sk (type: int), ss_net_paid_inc_tax (type: float), ss_net_profit (type: float)
@@ -920,10 +920,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss_n0
- filterExpr: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ filterExpr: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ predicate: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ss_net_paid_inc_tax (type: float), ss_net_profit (type: float), ss_sold_date_sk (type: int)
@@ -1245,10 +1245,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss_orc
- filterExpr: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ filterExpr: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 24 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ predicate: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 24 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
keys: ss_sold_date_sk (type: int), ss_net_paid_inc_tax (type: float), ss_net_profit (type: float)
@@ -1517,10 +1517,10 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: ss_orc
- filterExpr: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ filterExpr: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 24 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: ((ss_sold_date_sk >= 2452617) and (ss_sold_date_sk <= 2452638)) (type: boolean)
+ predicate: ss_sold_date_sk BETWEEN 2452617 AND 2452638 (type: boolean)
Statistics: Num rows: 24 Data size: 288 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: ss_net_paid_inc_tax (type: float), ss_net_profit (type: float), ss_sold_date_sk (type: int)
diff --git a/ql/src/test/results/clientpositive/llap/groupingset_optimize_hive_28489.q.out b/ql/src/test/results/clientpositive/llap/groupingset_optimize_hive_28489.q.out
index c7741dbe3dc..5d0c17d370b 100644
--- a/ql/src/test/results/clientpositive/llap/groupingset_optimize_hive_28489.q.out
+++ b/ql/src/test/results/clientpositive/llap/groupingset_optimize_hive_28489.q.out
@@ -126,26 +126,26 @@ STAGE PLANS:
Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: col0 (type: int), col1 (type: int), col2 (type: int), col3 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col5
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: _col2 (type: int)
+ Map-reduce partition columns: _col0 (type: int)
Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: int), _col5 (type: int)
+ value expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col2 (type: int), VALUE._col3 (type: int), VALUE._col4 (type: int), VALUE._col5 (type: int)
- outputColumnNames: _col2, _col3, _col4, _col5
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: int), VALUE._col2 (type: int), VALUE._col3 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 3 Data size: 48 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- aggregations: sum(_col5)
- keys: _col2 (type: int), _col3 (type: int), _col4 (type: int), 0L (type: bigint)
+ aggregations: sum(_col3)
+ keys: _col0 (type: int), _col1 (type: int), _col2 (type: int), 0L (type: bigint)
grouping sets: 0, 1, 3, 7
minReductionHashAggr: 0.4
mode: hash
diff --git a/ql/src/test/results/clientpositive/llap/implicit_cast_during_insert.q.out b/ql/src/test/results/clientpositive/llap/implicit_cast_during_insert.q.out
index 10c325737ac..f32f1a617e5 100644
--- a/ql/src/test/results/clientpositive/llap/implicit_cast_during_insert.q.out
+++ b/ql/src/test/results/clientpositive/llap/implicit_cast_during_insert.q.out
@@ -40,23 +40,23 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: src
- filterExpr: (key) IN (0, 1) (type: boolean)
+ filterExpr: (UDFToDouble(key)) IN (0.0D, 1.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key) IN (0, 1) (type: boolean)
- Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key)) IN (0.0D, 1.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: value (type: string), key (type: string)
outputColumnNames: _col1, _col2
- Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col2 (type: string)
null sort order: z
sort order: +
Map-reduce partition columns: _col2 (type: string)
- Statistics: Num rows: 3 Data size: 534 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Execution mode: llap
+ Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
@@ -64,36 +64,40 @@ STAGE PLANS:
Select Operator
expressions: UDFToInteger(KEY.reducesinkkey0) (type: int), VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 3 Data size: 546 Basic stats: COMPLETE Column stats: COMPLETE
- File Output Operator
- compressed: false
- Statistics: Num rows: 3 Data size: 546 Basic stats: COMPLETE Column stats: COMPLETE
- table:
- input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
- output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
- serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
- name: default.implicit_cast_during_insert
+ Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
outputColumnNames: c1, c2, p1
- Statistics: Num rows: 3 Data size: 546 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(c1), max(c1), count(1), count(c1), compute_bit_vector_hll(c1), max(length(c2)), avg(COALESCE(length(c2),0)), count(c2), compute_bit_vector_hll(c2)
keys: p1 (type: string)
mode: complete
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9
- Statistics: Num rows: 2 Data size: 838 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 104750 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: 'LONG' (type: string), UDFToLong(_col1) (type: bigint), UDFToLong(_col2) (type: bigint), (_col3 - _col4) (type: bigint), COALESCE(ndv_compute_bit_vector(_col5),0) (type: bigint), _col5 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col6,0)) (type: bigint), COALESCE(_col7,0) (type: double), (_col3 - _col8) (type: bigint), COALESCE(ndv_compute_bit_vector(_col9),0) (type: bigint), _col9 (type: binary), _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12
- Statistics: Num rows: 2 Data size: 1234 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 154250 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 1234 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 154250 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ File Output Operator
+ compressed: false
+ Dp Sort State: PARTITION_SORTED
+ Statistics: Num rows: 250 Data size: 45500 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.implicit_cast_during_insert
Stage: Stage-2
Dependency Collection
diff --git a/ql/src/test/results/clientpositive/llap/infer_bucket_sort_num_buckets.q.out b/ql/src/test/results/clientpositive/llap/infer_bucket_sort_num_buckets.q.out
index 9cb5fab2e72..f715d92f612 100644
--- a/ql/src/test/results/clientpositive/llap/infer_bucket_sort_num_buckets.q.out
+++ b/ql/src/test/results/clientpositive/llap/infer_bucket_sort_num_buckets.q.out
@@ -53,7 +53,7 @@ STAGE PLANS:
Statistics: Num rows: 1000 Data size: 178000 Basic stats: COMPLETE Column stats: COMPLETE
GatherStats: false
Select Operator
- expressions: if(((key % 3) < 2), 0, 1) (type: int), value (type: string), UDFToInteger((key % 2)) (type: int)
+ expressions: if(((UDFToDouble(key) % 3.0D) < 2.0D), 0, 1) (type: int), value (type: string), UDFToInteger((UDFToDouble(key) % 2.0D)) (type: int)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 1000 Data size: 99000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
@@ -168,23 +168,23 @@ STAGE PLANS:
value expressions: _col0 (type: int), _col1 (type: string)
auto parallelism: false
Select Operator
- expressions: _col0 (type: int), _col1 (type: string), _col2 (type: string)
- outputColumnNames: key, value, hr
- Statistics: Num rows: 1000 Data size: 279000 Basic stats: COMPLETE Column stats: COMPLETE
+ expressions: _col0 (type: int), _col1 (type: string), '2008-04-08' (type: string), _col2 (type: string)
+ outputColumnNames: key, value, ds, hr
+ Statistics: Num rows: 1000 Data size: 373000 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: min(key), max(key), count(1), count(key), compute_bit_vector_hll(key), max(length(value)), avg(COALESCE(length(value),0)), count(value), compute_bit_vector_hll(value)
- keys: '2008-04-08' (type: string), hr (type: string)
+ keys: ds (type: string), hr (type: string)
minReductionHashAggr: 0.684
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 316 Data size: 214248 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
bucketingVersion: 2
- key expressions: '2008-04-08' (type: string), _col1 (type: string)
+ key expressions: _col0 (type: string), _col1 (type: string)
null sort order: zz
numBuckets: -1
sort order: ++
- Map-reduce partition columns: '2008-04-08' (type: string), _col1 (type: string)
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 316 Data size: 214248 Basic stats: COMPLETE Column stats: COMPLETE
tag: -1
value expressions: _col2 (type: int), _col3 (type: int), _col4 (type: bigint), _col5 (type: bigint), _col6 (type: binary), _col7 (type: int), _col8 (type: struct<count:bigint,sum:double,input:int>), _col9 (type: bigint), _col10 (type: binary)
@@ -232,12 +232,12 @@ STAGE PLANS:
Reduce Operator Tree:
Group By Operator
aggregations: min(VALUE._col0), max(VALUE._col1), count(VALUE._col2), count(VALUE._col3), compute_bit_vector_hll(VALUE._col4), max(VALUE._col5), avg(VALUE._col6), count(VALUE._col7), compute_bit_vector_hll(VALUE._col8)
- keys: '2008-04-08' (type: string), KEY._col1 (type: string)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Statistics: Num rows: 316 Data size: 192760 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: 'LONG' (type: string), UDFToLong(_col2) (type: bigint), UDFToLong(_col3) (type: bigint), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col7,0)) (type: bigint), COALESCE(_col8,0) (type: double), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), '2008-04-08' (type: string), _col1 (type: string)
+ expressions: 'LONG' (type: string), UDFToLong(_col2) (type: bigint), UDFToLong(_col3) (type: bigint), (_col4 - _col5) (type: bigint), COALESCE(ndv_compute_bit_vector(_col6),0) (type: bigint), _col6 (type: binary), 'STRING' (type: string), UDFToLong(COALESCE(_col7,0)) (type: bigint), COALESCE(_col8,0) (type: double), (_col4 - _col9) (type: bigint), COALESCE(ndv_compute_bit_vector(_col10),0) (type: bigint), _col10 (type: binary), _col0 (type: string), _col1 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Statistics: Num rows: 316 Data size: 255328 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
diff --git a/ql/src/test/results/clientpositive/llap/input_columnarserde.q.out b/ql/src/test/results/clientpositive/llap/input_columnarserde.q.out
index 57c8c8eccc9..80957efa865 100644
--- a/ql/src/test/results/clientpositive/llap/input_columnarserde.q.out
+++ b/ql/src/test/results/clientpositive/llap/input_columnarserde.q.out
@@ -54,7 +54,7 @@ STAGE PLANS:
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: 1 (type: int)
+ Map-reduce partition columns: _col0 (type: array<int>)
Statistics: Num rows: 11 Data size: 34628 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<int>), _col1 (type: array<string>), _col2 (type: map<string,string>), _col3 (type: int), _col4 (type: string)
Execution mode: vectorized, llap
diff --git a/ql/src/test/results/clientpositive/llap/input_lazyserde.q.out b/ql/src/test/results/clientpositive/llap/input_lazyserde.q.out
index 3d692c27499..a8b8dc9d161 100644
--- a/ql/src/test/results/clientpositive/llap/input_lazyserde.q.out
+++ b/ql/src/test/results/clientpositive/llap/input_lazyserde.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: 1 (type: int)
+ Map-reduce partition columns: _col0 (type: array<int>)
Statistics: Num rows: 11 Data size: 34628 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<int>), _col1 (type: array<string>), _col2 (type: map<string,string>), _col3 (type: int), _col4 (type: string)
Execution mode: llap
diff --git a/ql/src/test/results/clientpositive/llap/input_lazyserde2.q.out b/ql/src/test/results/clientpositive/llap/input_lazyserde2.q.out
index f520411563a..7d2e469c1ce 100644
--- a/ql/src/test/results/clientpositive/llap/input_lazyserde2.q.out
+++ b/ql/src/test/results/clientpositive/llap/input_lazyserde2.q.out
@@ -62,7 +62,7 @@ STAGE PLANS:
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: 1 (type: int)
+ Map-reduce partition columns: _col0 (type: array<int>)
Statistics: Num rows: 11 Data size: 34628 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: array<int>), _col1 (type: array<string>), _col2 (type: map<string,string>), _col3 (type: int), _col4 (type: string)
Execution mode: llap
diff --git a/ql/src/test/results/clientpositive/llap/mapreduce5.q.out b/ql/src/test/results/clientpositive/llap/mapreduce5.q.out
index eb1a241e322..b64db4c71aa 100644
--- a/ql/src/test/results/clientpositive/llap/mapreduce5.q.out
+++ b/ql/src/test/results/clientpositive/llap/mapreduce5.q.out
@@ -45,7 +45,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string)
+ expressions: key (type: string), UDFToInteger((UDFToDouble(key) / 10.0D)) (type: int), UDFToInteger((UDFToDouble(key) % 10.0D)) (type: int), value (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
diff --git a/ql/src/test/results/clientpositive/llap/mapreduce6.q.out b/ql/src/test/results/clientpositive/llap/mapreduce6.q.out
index b6117f59df2..a9013f3410b 100644
--- a/ql/src/test/results/clientpositive/llap/mapreduce6.q.out
+++ b/ql/src/test/results/clientpositive/llap/mapreduce6.q.out
@@ -45,7 +45,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: key (type: string), UDFToInteger((key / 10)) (type: int), UDFToInteger((key % 10)) (type: int), value (type: string)
+ expressions: key (type: string), UDFToInteger((UDFToDouble(key) / 10.0D)) (type: int), UDFToInteger((UDFToDouble(key) % 10.0D)) (type: int), value (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 500 Data size: 93000 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
diff --git a/ql/src/test/results/clientpositive/llap/ppd2.q.out b/ql/src/test/results/clientpositive/llap/ppd2.q.out
index 82ba14b2dbe..26660d33a92 100644
--- a/ql/src/test/results/clientpositive/llap/ppd2.q.out
+++ b/ql/src/test/results/clientpositive/llap/ppd2.q.out
@@ -348,35 +348,43 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -386,31 +394,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col2 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col2 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -473,25 +477,28 @@ STAGE PLANS:
TableScan
alias: a
filterExpr: ((ds = '2008-04-08') and (hr = '11')) (type: boolean)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- Group By Operator
- aggregations: count(value)
- keys: key (type: string)
- minReductionHashAggr: 0.4
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 273000 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: ((ds = '2008-04-08') and (hr = '11')) (type: boolean)
+ Statistics: Num rows: 500 Data size: 273000 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: key, value
+ Statistics: Num rows: 500 Data size: 273000 Basic stats: COMPLETE Column stats: COMPLETE
+ Group By Operator
+ aggregations: count(value)
+ keys: key (type: string)
+ minReductionHashAggr: 0.4
+ mode: hash
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col1 (type: bigint)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: bigint)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -503,12 +510,12 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 316 Data size: 30020 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Filter Operator
- predicate: (_col1 > 1L) (type: boolean)
- Statistics: Num rows: 105 Data size: 9975 Basic stats: COMPLETE Column stats: COMPLETE
+ Filter Operator
+ predicate: (_col1 > 1L) (type: boolean)
+ Statistics: Num rows: 105 Data size: 9975 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
File Output Operator
compressed: false
Statistics: Num rows: 105 Data size: 9975 Basic stats: COMPLETE Column stats: COMPLETE
diff --git a/ql/src/test/results/clientpositive/llap/ppd_clusterby.q.out b/ql/src/test/results/clientpositive/llap/ppd_clusterby.q.out
index cdf98735542..a00aa6c0eea 100644
--- a/ql/src/test/results/clientpositive/llap/ppd_clusterby.q.out
+++ b/ql/src/test/results/clientpositive/llap/ppd_clusterby.q.out
@@ -24,21 +24,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 10) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 10.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 10) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 10.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -48,10 +48,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -99,35 +99,43 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -137,34 +145,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Filter Operator
- predicate: (_col0 = 20) (type: boolean)
- Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col2 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col2 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 265 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -211,21 +212,21 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 10) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 10.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 10) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 10.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
key expressions: _col0 (type: string)
null sort order: a
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
@@ -235,10 +236,10 @@ STAGE PLANS:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
@@ -286,35 +287,43 @@ STAGE PLANS:
Map Operator Tree:
TableScan
alias: x
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 356 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44500 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Map 4
Map Operator Tree:
TableScan
alias: y
- filterExpr: (key = 20) (type: boolean)
+ filterExpr: (UDFToDouble(key) = 20.0D) (type: boolean)
Statistics: Num rows: 500 Data size: 43500 Basic stats: COMPLETE Column stats: COMPLETE
Filter Operator
- predicate: (key = 20) (type: boolean)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 2 Data size: 174 Basic stats: COMPLETE Column stats: COMPLETE
+ predicate: (UDFToDouble(key) = 20.0D) (type: boolean)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 21750 Basic stats: COMPLETE Column stats: COMPLETE
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -324,31 +333,27 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 key (type: string)
- 1 key (type: string)
- outputColumnNames: _col0, _col1, _col6
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string), _col6 (type: string)
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- Reduce Output Operator
- key expressions: _col1 (type: string)
- null sort order: a
- sort order: +
- Map-reduce partition columns: _col1 (type: string)
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: string), _col2 (type: string)
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ null sort order: a
+ sort order: +
+ Map-reduce partition columns: _col1 (type: string)
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col0 (type: string), _col2 (type: string)
Reducer 3
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), VALUE._col1 (type: string)
outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
compressed: false
- Statistics: Num rows: 2 Data size: 530 Basic stats: COMPLETE Column stats: COMPLETE
+ Statistics: Num rows: 250 Data size: 66250 Basic stats: COMPLETE Column stats: COMPLETE
table:
input format: org.apache.hadoop.mapred.SequenceFileInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_exclude_join.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_exclude_join.q.out
index b142591b019..2b8313248b0 100644
--- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_exclude_join.q.out
+++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_exclude_join.q.out
@@ -39,13 +39,13 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
value expressions: _col1 (type: string)
- Reduce Output Operator
- key expressions: key (type: string)
- null sort order: z
- sort order: +
- Map-reduce partition columns: key (type: string)
- Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ null sort order: z
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
@@ -60,7 +60,7 @@ STAGE PLANS:
Inner Join 0 to 1
keys:
0 _col0 (type: string)
- 1 key (type: string)
+ 1 _col0 (type: string)
outputColumnNames: _col0, _col1, _col2, _col3
input vertices:
1 Reducer 3
@@ -80,14 +80,14 @@ STAGE PLANS:
Reduce Operator Tree:
Select Operator
expressions: KEY.reducesinkkey0 (type: string), VALUE._col0 (type: string)
- outputColumnNames: key, value
+ outputColumnNames: _col0, _col1
Reduce Output Operator
- key expressions: key (type: string)
+ key expressions: _col0 (type: string)
null sort order: z
sort order: +
- Map-reduce partition columns: key (type: string)
+ Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 500 Data size: 89000 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: value (type: string)
+ value expressions: _col1 (type: string)
Stage: Stage-0
Fetch Operator
diff --git a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_stat_based.q.out b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_stat_based.q.out
index 31d3775b733..0d51eff4152 100644
--- a/ql/src/test/results/clientpositive/llap/reduce_deduplicate_stat_based.q.out
+++ b/ql/src/test/results/clientpositive/llap/reduce_deduplicate_stat_based.q.out
@@ -145,26 +145,26 @@ STAGE PLANS:
Statistics: Num rows: 741000000 Data size: 5927996004 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: id (type: int), country (type: int)
- outputColumnNames: _col0, _col3
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 741000000 Data size: 5927996004 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: _col3 (type: int), _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: int), _col0 (type: int)
Statistics: Num rows: 741000000 Data size: 5927996004 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: int), _col3 (type: int)
+ value expressions: _col0 (type: int), _col1 (type: int)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: int), VALUE._col3 (type: int)
- outputColumnNames: _col0, _col3
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: int)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 741000000 Data size: 5927996004 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
- keys: _col3 (type: int)
+ keys: _col1 (type: int)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
@@ -227,26 +227,26 @@ STAGE PLANS:
Statistics: Num rows: 741000000 Data size: 65864000085 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: id (type: int), sex (type: string)
- outputColumnNames: _col0, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 741000000 Data size: 65864000085 Basic stats: COMPLETE Column stats: COMPLETE
Reduce Output Operator
null sort order:
sort order:
- Map-reduce partition columns: _col2 (type: string), _col0 (type: int)
+ Map-reduce partition columns: _col1 (type: string), _col0 (type: int)
Statistics: Num rows: 741000000 Data size: 65864000085 Basic stats: COMPLETE Column stats: COMPLETE
- value expressions: _col0 (type: int), _col2 (type: string)
+ value expressions: _col0 (type: int), _col1 (type: string)
Execution mode: vectorized, llap
LLAP IO: all inputs
Reducer 2
Execution mode: vectorized, llap
Reduce Operator Tree:
Select Operator
- expressions: VALUE._col0 (type: int), VALUE._col2 (type: string)
- outputColumnNames: _col0, _col2
+ expressions: VALUE._col0 (type: int), VALUE._col1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 741000000 Data size: 65864000085 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
aggregations: count(_col0)
- keys: _col2 (type: string)
+ keys: _col1 (type: string)
minReductionHashAggr: 0.99
mode: hash
outputColumnNames: _col0, _col1
diff --git a/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out b/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out
index 1b661ebff2e..ca5dcdd470e 100644
--- a/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out
+++ b/ql/src/test/results/clientpositive/llap/reducesink_dedup.q.out
@@ -10,31 +10,31 @@ distribute by 1 sort by 1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@part
#### A masked pattern was here ####
-almond antique burnished rose metallic
-almond antique burnished rose metallic
-almond antique chartreuse lavender yellow
+almond antique medium spring khaki
+almond antique metallic orange dim
almond antique salmon chartreuse burlywood
-almond aquamarine burnished black steel
-almond aquamarine pink moccasin thistle
-almond antique violet chocolate turquoise
-almond antique violet turquoise frosted
almond aquamarine midnight light salmon
-almond aquamarine rose maroon antique
-almond aquamarine sandy cyan gainsboro
+almond aquamarine pink moccasin thistle
+almond aquamarine yellow dodger mint
+almond azure aquamarine papaya violet
+almond antique blue firebrick mint
+almond antique burnished rose metallic
+almond antique burnished rose metallic
almond antique chartreuse khaki white
+almond antique chartreuse lavender yellow
almond antique forest lavender goldenrod
-almond antique metallic orange dim
+almond antique gainsboro frosted violet
almond antique misty red olive
almond antique olive coral navajo
-almond antique gainsboro frosted violet
-almond antique violet mint lemon
-almond aquamarine floral ivory bisque
-almond aquamarine yellow dodger mint
-almond azure aquamarine papaya violet
-almond antique blue firebrick mint
-almond antique medium spring khaki
almond antique sky peru orange
+almond antique violet chocolate turquoise
+almond antique violet mint lemon
+almond antique violet turquoise frosted
+almond aquamarine burnished black steel
almond aquamarine dodger light gainsboro
+almond aquamarine floral ivory bisque
+almond aquamarine rose maroon antique
+almond aquamarine sandy cyan gainsboro
almond azure blanched chiffon midnight
PREHOOK: query: create temporary table d1 (key int)
PREHOOK: type: CREATETABLE