You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2015/01/09 08:41:19 UTC
svn commit: r1650453 [8/9] - in /hive/branches/spark: ./
common/src/java/org/apache/hadoop/hive/conf/
hbase-handler/src/java/org/apache/hadoop/hive/hbase/
hcatalog/server-extensions/src/main/java/org/apache/hive/hcatalog/listener/
hcatalog/server-exten...
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java Fri Jan 9 07:41:17 2015
@@ -22,6 +22,8 @@ import java.lang.reflect.InvocationTarge
import java.lang.reflect.UndeclaredThrowableException;
import java.math.BigDecimal;
import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -66,6 +68,8 @@ import org.apache.calcite.rel.rules.Filt
import org.apache.calcite.rel.rules.JoinPushTransitivePredicatesRule;
import org.apache.calcite.rel.rules.JoinToMultiJoinRule;
import org.apache.calcite.rel.rules.LoptOptimizeJoinRule;
+import org.apache.calcite.rel.rules.ProjectRemoveRule;
+import org.apache.calcite.rel.rules.ReduceExpressionsRule;
import org.apache.calcite.rel.rules.SemiJoinFilterTransposeRule;
import org.apache.calcite.rel.rules.SemiJoinJoinTransposeRule;
import org.apache.calcite.rel.rules.SemiJoinProjectTransposeRule;
@@ -106,15 +110,16 @@ import org.apache.hadoop.hive.ql.exec.Op
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
+import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveCalciteUtil;
+import org.apache.hadoop.hive.ql.optimizer.calcite.HiveDefaultRelMetadataProvider;
import org.apache.hadoop.hive.ql.optimizer.calcite.HiveTypeSystemImpl;
-import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.TraitsUtil;
import org.apache.hadoop.hive.ql.optimizer.calcite.cost.HiveVolcanoPlanner;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveAggregate;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveFilter;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveJoin;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveRelNode;
@@ -159,12 +164,12 @@ import org.apache.hadoop.hive.serde2.typ
import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Lists;
-import com.google.common.collect.ImmutableList.Builder;
public class CalcitePlanner extends SemanticAnalyzer {
- private AtomicInteger noColsMissingStats = new AtomicInteger(0);
+ private final AtomicInteger noColsMissingStats = new AtomicInteger(0);
private List<FieldSchema> topLevelFieldSchema;
private SemanticException semanticException;
private boolean runCBO = true;
@@ -187,6 +192,7 @@ public class CalcitePlanner extends Sema
}
}
+ @Override
@SuppressWarnings("rawtypes")
Operator genOPTree(ASTNode ast, PlannerContext plannerCtx) throws SemanticException {
Operator sinkOp = null;
@@ -282,7 +288,7 @@ public class CalcitePlanner extends Sema
/**
* Can CBO handle the given AST?
- *
+ *
* @param ast
* Top level AST
* @param qb
@@ -290,7 +296,7 @@ public class CalcitePlanner extends Sema
* @param cboCtx
* @param semAnalyzer
* @return boolean
- *
+ *
* Assumption:<br>
* If top level QB is query then everything below it must also be
* Query.
@@ -345,7 +351,7 @@ public class CalcitePlanner extends Sema
/**
* Checks whether Calcite can handle the query.
- *
+ *
* @param queryProperties
* @param conf
* @param topLevelQB
@@ -354,7 +360,7 @@ public class CalcitePlanner extends Sema
* Whether return value should be verbose in case of failure.
* @return null if the query can be handled; non-null reason string if it
* cannot be.
- *
+ *
* Assumption:<br>
* 1. If top level QB is query then everything below it must also be
* Query<br>
@@ -548,7 +554,7 @@ public class CalcitePlanner extends Sema
/**
* Get Optimized AST for the given QB tree in the semAnalyzer.
- *
+ *
* @return Optimized operator tree translated in to Hive AST
* @throws SemanticException
*/
@@ -572,7 +578,7 @@ public class CalcitePlanner extends Sema
/***
* Unwraps Calcite Invocation exceptions coming meta data provider chain and
* obtains the real cause.
- *
+ *
* @param Exception
*/
private void rethrowCalciteException(Exception e) throws SemanticException {
@@ -651,7 +657,7 @@ public class CalcitePlanner extends Sema
private class CalcitePlannerAction implements Frameworks.PlannerAction<RelNode> {
private RelOptCluster cluster;
private RelOptSchema relOptSchema;
- private Map<String, PrunedPartitionList> partitionCache;
+ private final Map<String, PrunedPartitionList> partitionCache;
// TODO: Do we need to keep track of RR, ColNameToPosMap for every op or
// just last one.
@@ -706,6 +712,11 @@ public class CalcitePlanner extends Sema
hepPgmBldr.addRuleInstance(new LoptOptimizeJoinRule(HiveJoin.HIVE_JOIN_FACTORY,
HiveProject.DEFAULT_PROJECT_FACTORY, HiveFilter.DEFAULT_FILTER_FACTORY));
+ hepPgmBldr.addRuleInstance(ReduceExpressionsRule.JOIN_INSTANCE);
+ hepPgmBldr.addRuleInstance(ReduceExpressionsRule.FILTER_INSTANCE);
+ hepPgmBldr.addRuleInstance(ReduceExpressionsRule.PROJECT_INSTANCE);
+
+ hepPgmBldr.addRuleInstance(ProjectRemoveRule.INSTANCE);
hepPgm = hepPgmBldr.build();
HepPlanner hepPlanner = new HepPlanner(hepPgm);
@@ -736,7 +747,7 @@ public class CalcitePlanner extends Sema
/**
* Perform all optimizations before Join Ordering.
- *
+ *
* @param basePlan
* original plan
* @param mdProvider
@@ -754,7 +765,11 @@ public class CalcitePlanner extends Sema
SemiJoinFilterTransposeRule.INSTANCE, SemiJoinProjectTransposeRule.INSTANCE);
// 2. PPD
- basePlan = hepPlan(basePlan, true, mdProvider, new HiveFilterProjectTransposeRule(
+ basePlan = hepPlan(basePlan, true, mdProvider,
+ ReduceExpressionsRule.PROJECT_INSTANCE,
+ ReduceExpressionsRule.FILTER_INSTANCE,
+ ReduceExpressionsRule.JOIN_INSTANCE,
+ new HiveFilterProjectTransposeRule(
Filter.class, HiveFilter.DEFAULT_FILTER_FACTORY, HiveProject.class,
HiveProject.DEFAULT_PROJECT_FACTORY), new HiveFilterSetOpTransposeRule(
HiveFilter.DEFAULT_FILTER_FACTORY),
@@ -787,7 +802,7 @@ public class CalcitePlanner extends Sema
/**
* Run the HEP Planner with the given rule set.
- *
+ *
* @param basePlan
* @param followPlanChanges
* @param mdProvider
@@ -1057,7 +1072,7 @@ public class CalcitePlanner extends Sema
/**
* Generate Join Logical Plan Relnode by walking through the join AST.
- *
+ *
* @param qb
* @param aliasToRel
* Alias(Table/Relation alias) to RelNode; only read and not
@@ -1276,7 +1291,7 @@ public class CalcitePlanner extends Sema
Map<String, RelNode> aliasToRel, boolean forHavingClause) throws SemanticException {
/*
* Handle Subquery predicates.
- *
+ *
* Notes (8/22/14 hb): Why is this a copy of the code from {@link
* #genFilterPlan} - for now we will support the same behavior as non CBO
* route. - but plan to allow nested SubQueries(Restriction.9.m) and
@@ -1509,7 +1524,7 @@ public class CalcitePlanner extends Sema
}
private RelNode genGBRelNode(List<ExprNodeDesc> gbExprs, List<AggInfo> aggInfoLst,
- RelNode srcRel) throws SemanticException {
+ List<Integer> groupSets, RelNode srcRel) throws SemanticException {
ImmutableMap<String, Integer> posMap = this.relToHiveColNameCalcitePosMap.get(srcRel);
RexNodeConverter converter = new RexNodeConverter(this.cluster, srcRel.getRowType(), posMap,
0, false);
@@ -1541,10 +1556,25 @@ public class CalcitePlanner extends Sema
}
RelNode gbInputRel = HiveProject.create(srcRel, gbChildProjLst, null);
+ // Grouping sets: we need to transform them into ImmutableBitSet
+ // objects for Calcite
+ List<ImmutableBitSet> transformedGroupSets = null;
+ if(groupSets != null && !groupSets.isEmpty()) {
+ Set<ImmutableBitSet> setTransformedGroupSets =
+ new HashSet<ImmutableBitSet>(groupSets.size());
+ for(int val: groupSets) {
+ setTransformedGroupSets.add(convert(val));
+ }
+ // Calcite expects the grouping sets sorted and without duplicates
+ transformedGroupSets = new ArrayList<ImmutableBitSet>(setTransformedGroupSets);
+ Collections.sort(transformedGroupSets, ImmutableBitSet.COMPARATOR);
+ }
+
HiveRelNode aggregateRel = null;
try {
aggregateRel = new HiveAggregate(cluster, cluster.traitSetOf(HiveRelNode.CONVENTION),
- gbInputRel, false, groupSet, null, aggregateCalls);
+ gbInputRel, (transformedGroupSets!=null ? true:false), groupSet,
+ transformedGroupSets, aggregateCalls);
} catch (InvalidRelException e) {
throw new SemanticException(e);
}
@@ -1552,6 +1582,19 @@ public class CalcitePlanner extends Sema
return aggregateRel;
}
+ private ImmutableBitSet convert(int value) {
+ BitSet bits = new BitSet();
+ int index = 0;
+ while (value != 0L) {
+ if (value % 2 != 0) {
+ bits.set(index);
+ }
+ ++index;
+ value = value >>> 1;
+ }
+ return ImmutableBitSet.FROM_BIT_SET.apply(bits);
+ }
+
private void addAlternateGByKeyMappings(ASTNode gByExpr, ColumnInfo colInfo,
RowResolver gByInputRR, RowResolver gByRR) {
if (gByExpr.getType() == HiveParser.DOT
@@ -1676,7 +1719,7 @@ public class CalcitePlanner extends Sema
/**
* Generate GB plan.
- *
+ *
* @param qb
* @param srcRel
* @return TODO: 1. Grouping Sets (roll up..)
@@ -1686,26 +1729,6 @@ public class CalcitePlanner extends Sema
RelNode gbRel = null;
QBParseInfo qbp = getQBParseInfo(qb);
- // 0. for GSets, Cube, Rollup, bail from Calcite path.
- if (!qbp.getDestRollups().isEmpty() || !qbp.getDestGroupingSets().isEmpty()
- || !qbp.getDestCubes().isEmpty()) {
- String gbyClause = null;
- HashMap<String, ASTNode> gbysMap = qbp.getDestToGroupBy();
- if (gbysMap.size() == 1) {
- ASTNode gbyAST = gbysMap.entrySet().iterator().next().getValue();
- gbyClause = ctx.getTokenRewriteStream().toString(gbyAST.getTokenStartIndex(),
- gbyAST.getTokenStopIndex());
- gbyClause = "in '" + gbyClause + "'.";
- } else {
- gbyClause = ".";
- }
- String msg = String.format("Encountered Grouping Set/Cube/Rollup%s"
- + " Currently we don't support Grouping Set/Cube/Rollup" + " clauses in CBO,"
- + " turn off cbo for these queries.", gbyClause);
- LOG.debug(msg);
- throw new CalciteSemanticException(msg);
- }
-
// 1. Gather GB Expressions (AST) (GB + Aggregations)
// NOTE: Multi Insert is not supported
String detsClauseName = qbp.getClauseNames().iterator().next();
@@ -1739,18 +1762,34 @@ public class CalcitePlanner extends Sema
}
}
- // 4. Construct aggregation function Info
+ // 4. GroupingSets, Cube, Rollup
+ int groupingColsSize = gbExprNDescLst.size();
+ List<Integer> groupingSets = null;
+ if (!qbp.getDestRollups().isEmpty()
+ || !qbp.getDestGroupingSets().isEmpty()
+ || !qbp.getDestCubes().isEmpty()) {
+ if (qbp.getDestRollups().contains(detsClauseName)) {
+ groupingSets = getGroupingSetsForRollup(grpByAstExprs.size());
+ } else if (qbp.getDestCubes().contains(detsClauseName)) {
+ groupingSets = getGroupingSetsForCube(grpByAstExprs.size());
+ } else if (qbp.getDestGroupingSets().contains(detsClauseName)) {
+ groupingSets = getGroupingSets(grpByAstExprs, qbp, detsClauseName);
+ }
+ groupingColsSize = groupingColsSize * 2;
+ }
+
+ // 5. Construct aggregation function Info
ArrayList<AggInfo> aggregations = new ArrayList<AggInfo>();
if (hasAggregationTrees) {
assert (aggregationTrees != null);
for (ASTNode value : aggregationTrees.values()) {
- // 4.1 Determine type of UDAF
+ // 5.1 Determine type of UDAF
// This is the GenericUDAF name
String aggName = SemanticAnalyzer.unescapeIdentifier(value.getChild(0).getText());
boolean isDistinct = value.getType() == HiveParser.TOK_FUNCTIONDI;
boolean isAllColumns = value.getType() == HiveParser.TOK_FUNCTIONSTAR;
- // 4.2 Convert UDAF Params to ExprNodeDesc
+ // 5.2 Convert UDAF Params to ExprNodeDesc
ArrayList<ExprNodeDesc> aggParameters = new ArrayList<ExprNodeDesc>();
for (int i = 1; i < value.getChildCount(); i++) {
ASTNode paraExpr = (ASTNode) value.getChild(i);
@@ -1767,18 +1806,62 @@ public class CalcitePlanner extends Sema
aggParameters);
AggInfo aInfo = new AggInfo(aggParameters, udaf.returnType, aggName, isDistinct);
aggregations.add(aInfo);
- String field = SemanticAnalyzer.getColumnInternalName(gbExprNDescLst.size()
- + aggregations.size() - 1);
+ String field = getColumnInternalName(groupingColsSize + aggregations.size() - 1);
outputColumnNames.add(field);
groupByOutputRowResolver.putExpression(value, new ColumnInfo(field, aInfo.m_returnType,
"", false));
}
}
- gbRel = genGBRelNode(gbExprNDescLst, aggregations, srcRel);
+ gbRel = genGBRelNode(gbExprNDescLst, aggregations, groupingSets, srcRel);
relToHiveColNameCalcitePosMap.put(gbRel,
buildHiveToCalciteColumnMap(groupByOutputRowResolver, gbRel));
this.relToHiveRR.put(gbRel, groupByOutputRowResolver);
+
+ // 6. If GroupingSets, Cube, Rollup were used, we account grouping__id.
+ // Further, we insert a project operator on top to remove the grouping
+ // boolean associated to each column in Calcite; this will avoid
+ // recalculating all column positions when we go back from Calcite to Hive
+ if(groupingSets != null && !groupingSets.isEmpty()) {
+ RowResolver selectOutputRowResolver = new RowResolver();
+ selectOutputRowResolver.setIsExprResolver(true);
+ RowResolver.add(selectOutputRowResolver, groupByOutputRowResolver);
+ outputColumnNames = new ArrayList<String>(outputColumnNames);
+
+ // 6.1 List of columns to keep from groupBy operator
+ List<RelDataTypeField> gbOutput = gbRel.getRowType().getFieldList();
+ List<RexNode> calciteColLst = new ArrayList<RexNode>();
+ for(RelDataTypeField gbOut: gbOutput) {
+ if(gbOut.getIndex() < gbExprNDescLst.size() ||
+ gbOut.getIndex() >= gbExprNDescLst.size() * 2) {
+ calciteColLst.add(new RexInputRef(gbOut.getIndex(), gbOut.getType()));
+ }
+ }
+
+ // 6.2 Add column for grouping_id function
+ String field = getColumnInternalName(groupingColsSize + aggregations.size());
+ outputColumnNames.add(field);
+ selectOutputRowResolver.put(null, VirtualColumn.GROUPINGID.getName(),
+ new ColumnInfo(
+ field,
+ TypeInfoFactory.stringTypeInfo,
+ null,
+ true));
+
+ // 6.3 Compute column for grouping_id function in Calcite
+ List<RexNode> identifierCols = new ArrayList<RexNode>();
+ for(int i = gbExprNDescLst.size(); i < gbExprNDescLst.size() * 2; i++) {
+ identifierCols.add(new RexInputRef(
+ i, gbOutput.get(i).getType()));
+ }
+ final RexBuilder rexBuilder = cluster.getRexBuilder();
+ RexNode groupingID = rexBuilder.makeCall(HiveGroupingID.GROUPING__ID,
+ identifierCols);
+ calciteColLst.add(groupingID);
+
+ // Create select
+ gbRel = this.genSelectRelNode(calciteColLst, selectOutputRowResolver, gbRel);
+ }
}
return gbRel;
@@ -1788,7 +1871,7 @@ public class CalcitePlanner extends Sema
* Generate OB RelNode and input Select RelNode that should be used to
* introduce top constraining Project. If Input select RelNode is not
* present then don't introduce top constraining select.
- *
+ *
* @param qb
* @param srcRel
* @param outermostOB
@@ -2198,7 +2281,7 @@ public class CalcitePlanner extends Sema
/**
* NOTE: there can only be one select caluse since we don't handle multi
* destination insert.
- *
+ *
* @throws SemanticException
*/
private RelNode genSelectLogicalPlan(QB qb, RelNode srcRel, RelNode starSrcRel)
Modified: hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Jan 9 07:41:17 2015
@@ -3117,7 +3117,7 @@ public class SemanticAnalyzer extends Ba
}
}
- private List<Integer> getGroupingSetsForRollup(int size) {
+ protected List<Integer> getGroupingSetsForRollup(int size) {
List<Integer> groupingSetKeys = new ArrayList<Integer>();
for (int i = 0; i <= size; i++) {
groupingSetKeys.add((1 << i) - 1);
@@ -3125,7 +3125,7 @@ public class SemanticAnalyzer extends Ba
return groupingSetKeys;
}
- private List<Integer> getGroupingSetsForCube(int size) {
+ protected List<Integer> getGroupingSetsForCube(int size) {
int count = 1 << size;
List<Integer> results = new ArrayList<Integer>(count);
for (int i = 0; i < count; ++i) {
@@ -3152,7 +3152,7 @@ public class SemanticAnalyzer extends Ba
return new ObjectPair<List<ASTNode>, List<Integer>>(groupByExprs, groupingSets);
}
- private List<Integer> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo,
+ protected List<Integer> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo,
String dest) throws SemanticException {
Map<String, Integer> exprPos = new HashMap<String, Integer>();
for (int i = 0; i < groupByExpr.size(); ++i) {
Modified: hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java (original)
+++ hive/branches/spark/ql/src/test/org/apache/hadoop/hive/ql/io/TestHiveBinarySearchRecordReader.java Fri Jan 9 07:41:17 2015
@@ -118,7 +118,7 @@ public class TestHiveBinarySearchRecordR
conf.set(Utilities.INPUT_NAME, "TestHiveBinarySearchRecordReader");
ioContext = IOContext.get(conf);
ioContext.setUseSorted(false);
- ioContext.setIsBinarySearching(false);
+ ioContext.setBinarySearching(false);
ioContext.setEndBinarySearch(false);
ioContext.setComparison(null);
ioContext.setGenericUDFClassName(null);
@@ -252,7 +252,7 @@ public class TestHiveBinarySearchRecordR
ioContext.setGenericUDFClassName(GenericUDFOPEqual.class.getName());
Assert.assertTrue(ioContext.isBinarySearching());
Assert.assertTrue(executeDoNext(hbsReader));
- ioContext.setIsBinarySearching(false);
+ ioContext.setBinarySearching(false);
ioContext.setComparison(-1);
Assert.assertTrue(executeDoNext(hbsReader));
ioContext.setComparison(0);
@@ -292,7 +292,7 @@ public class TestHiveBinarySearchRecordR
ioContext.setGenericUDFClassName(GenericUDFOPGreaterThan.class.getName());
Assert.assertTrue(ioContext.isBinarySearching());
Assert.assertTrue(executeDoNext(hbsReader));
- ioContext.setIsBinarySearching(false);
+ ioContext.setBinarySearching(false);
ioContext.setComparison(-1);
Assert.assertTrue(executeDoNext(hbsReader));
ioContext.setComparison(0);
@@ -306,7 +306,7 @@ public class TestHiveBinarySearchRecordR
ioContext.setGenericUDFClassName(GenericUDFOPEqualOrGreaterThan.class.getName());
Assert.assertTrue(ioContext.isBinarySearching());
Assert.assertTrue(executeDoNext(hbsReader));
- ioContext.setIsBinarySearching(false);
+ ioContext.setBinarySearching(false);
ioContext.setComparison(-1);
Assert.assertTrue(executeDoNext(hbsReader));
ioContext.setComparison(0);
Modified: hive/branches/spark/ql/src/test/queries/clientpositive/groupby_cube1.q
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/queries/clientpositive/groupby_cube1.q?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/queries/clientpositive/groupby_cube1.q (original)
+++ hive/branches/spark/ql/src/test/queries/clientpositive/groupby_cube1.q Fri Jan 9 07:41:17 2015
@@ -13,6 +13,11 @@ SELECT key, val, count(1) FROM T1 GROUP
SELECT key, val, count(1) FROM T1 GROUP BY key, val with cube;
EXPLAIN
+SELECT key, val, GROUPING__ID, count(1) FROM T1 GROUP BY key, val with cube;
+
+SELECT key, val, GROUPING__ID, count(1) FROM T1 GROUP BY key, val with cube;
+
+EXPLAIN
SELECT key, count(distinct val) FROM T1 GROUP BY key with cube;
SELECT key, count(distinct val) FROM T1 GROUP BY key with cube;
Modified: hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby.q.out Fri Jan 9 07:41:17 2015
@@ -373,10 +373,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE
@@ -430,10 +430,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE
@@ -487,10 +487,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 8 Data size: 1400 Basic stats: COMPLETE Column stats: COMPLETE
@@ -544,10 +544,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE
@@ -601,10 +601,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 24 Data size: 4200 Basic stats: COMPLETE Column stats: COMPLETE
@@ -658,10 +658,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 32 Data size: 5600 Basic stats: COMPLETE Column stats: COMPLETE
@@ -776,10 +776,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 16 Data size: 2800 Basic stats: COMPLETE Column stats: COMPLETE
@@ -892,10 +892,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE
@@ -949,10 +949,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 24 Data size: 2388 Basic stats: COMPLETE Column stats: NONE
@@ -1006,10 +1006,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
@@ -1063,10 +1063,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 16 Data size: 1592 Basic stats: COMPLETE Column stats: NONE
@@ -1120,10 +1120,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 24 Data size: 2388 Basic stats: COMPLETE Column stats: NONE
@@ -1177,10 +1177,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE
@@ -1291,10 +1291,10 @@ STAGE PLANS:
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), locid (type: int)
- outputColumnNames: state, locid
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 8 Data size: 796 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), locid (type: int), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: int), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 32 Data size: 3184 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/annotate_stats_groupby2.q.out Fri Jan 9 07:41:17 2015
@@ -148,10 +148,10 @@ STAGE PLANS:
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: state (type: string), country (type: string)
- outputColumnNames: state, country
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- keys: state (type: string), country (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 80 Data size: 800 Basic stats: COMPLETE Column stats: NONE
@@ -325,10 +325,10 @@ STAGE PLANS:
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), country (type: string)
- outputColumnNames: state, country
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), country (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 32 Data size: 8256 Basic stats: COMPLETE Column stats: COMPLETE
@@ -439,10 +439,10 @@ STAGE PLANS:
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
expressions: state (type: string), country (type: string)
- outputColumnNames: state, country
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 20 Data size: 200 Basic stats: COMPLETE Column stats: COMPLETE
Group By Operator
- keys: state (type: string), country (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 80 Data size: 20640 Basic stats: COMPLETE Column stats: COMPLETE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join16.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join16.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join16.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join16.q.out Fri Jan 9 07:41:17 2015
@@ -23,11 +23,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:a
+ $hdt$_0:$hdt$_0:$hdt$_0:a
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:a
+ $hdt$_0:$hdt$_0:$hdt$_0:a
TableScan
alias: a
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join17.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join17.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join17.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join17.q.out Fri Jan 9 07:41:17 2015
@@ -24,11 +24,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join19.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join19.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join19.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join19.q.out Fri Jan 9 07:41:17 2015
@@ -26,11 +26,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:src2
+ $hdt$_1:src2
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:src2
+ $hdt$_1:src2
TableScan
alias: src2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join2.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join2.q.out Fri Jan 9 07:41:17 2015
@@ -24,14 +24,14 @@ STAGE PLANS:
Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
Fetch Operator
limit: -1
- $hdt$_0:$hdt$_2:src1
+ $hdt$_2:src1
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -46,7 +46,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- $hdt$_0:$hdt$_2:src1
+ $hdt$_2:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join22.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join22.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join22.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join22.q.out Fri Jan 9 07:41:17 2015
@@ -13,14 +13,14 @@ STAGE PLANS:
Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:src4
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:src4
Fetch Operator
limit: -1
$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:src4
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:src4
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:src4
TableScan
alias: src4
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join27.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join27.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join27.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join27.q.out Fri Jan 9 07:41:17 2015
@@ -73,11 +73,11 @@ STAGE PLANS:
Stage: Stage-7
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_1:$hdt$_1:src
+ $hdt$_0:$hdt$_1:$hdt$_1:src
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_1:$hdt$_1:src
+ $hdt$_0:$hdt$_1:$hdt$_1:src
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join3.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join3.q.out Fri Jan 9 07:41:17 2015
@@ -24,14 +24,14 @@ STAGE PLANS:
Stage: Stage-6
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
Fetch Operator
limit: -1
- $hdt$_0:$hdt$_1:src1
+ $hdt$_1:src1
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -47,7 +47,7 @@ STAGE PLANS:
0 _col0 (type: string)
1 _col0 (type: string)
2 _col0 (type: string)
- $hdt$_0:$hdt$_1:src1
+ $hdt$_1:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join33.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join33.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join33.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join33.q.out Fri Jan 9 07:41:17 2015
@@ -25,11 +25,11 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:src
+ $hdt$_0:src
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:src
+ $hdt$_0:src
TableScan
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join7.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join7.q.out Fri Jan 9 07:41:17 2015
@@ -48,10 +48,8 @@ INSERT OVERWRITE TABLE dest1 SELECT c.c1
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
Stage-1 is a root stage
- Stage-7 depends on stages: Stage-1
- Stage-6 depends on stages: Stage-7
- Stage-0 depends on stages: Stage-6
- Stage-3 depends on stages: Stage-0
+ Stage-0 depends on stages: Stage-1
+ Stage-2 depends on stages: Stage-0
STAGE PLANS:
Stage: Stage-1
@@ -89,30 +87,6 @@ STAGE PLANS:
Map-reduce partition columns: _col0 (type: string)
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: string)
- Reduce Operator Tree:
- Join Operator
- condition map:
- Outer Join 0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 60 Data size: 642 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
-
- Stage: Stage-7
- Map Reduce Local Work
- Alias -> Map Local Tables:
- $hdt$_1:$hdt$_1:src1
- Fetch Operator
- limit: -1
- Alias -> Map Local Operator Tree:
- $hdt$_1:$hdt$_1:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -123,37 +97,35 @@ STAGE PLANS:
expressions: key (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
- HashTable Sink Operator
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
-
- Stage: Stage-6
- Map Reduce
- Map Operator Tree:
- TableScan
- Map Join Operator
- condition map:
- Left Outer Join0 to 1
- keys:
- 0 _col0 (type: string)
- 1 _col0 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int), _col3 (type: string), UDFToInteger(_col4) (type: int), _col5 (type: string)
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
- Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 66 Data size: 706 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
- name: default.dest1
- Local Work:
- Map Reduce Local Work
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 55 Data size: 584 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Outer Join 0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: UDFToInteger(_col0) (type: int), _col1 (type: string), UDFToInteger(_col2) (type: int), _col3 (type: string), UDFToInteger(_col4) (type: int), _col5 (type: string)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 121 Data size: 1284 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ name: default.dest1
Stage: Stage-0
Move Operator
@@ -165,7 +137,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.dest1
- Stage: Stage-3
+ Stage: Stage-2
Stats-Aggr Operator
PREHOOK: query: FROM (
@@ -211,7 +183,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@src
POSTHOOK: Output: default@dest1
POSTHOOK: Lineage: dest1.c1 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
-POSTHOOK: Lineage: dest1.c2 EXPRESSION [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
+POSTHOOK: Lineage: dest1.c2 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c3 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c4 SIMPLE [(src)src1.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c5 EXPRESSION [(src)src1.FieldSchema(name:key, type:string, comment:default), ]
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join8.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join8.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join8.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join8.q.out Fri Jan 9 07:41:17 2015
@@ -46,11 +46,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:src1
+ $hdt$_1:$hdt$_1:$hdt$_1:src1
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:src1
+ $hdt$_1:$hdt$_1:$hdt$_1:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join9.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join9.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join9.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join9.q.out Fri Jan 9 07:41:17 2015
@@ -24,11 +24,11 @@ STAGE PLANS:
Stage: Stage-5
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:src1
+ $hdt$_0:src1
TableScan
alias: src1
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join_nulls.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join_nulls.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join_nulls.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join_nulls.q.out Fri Jan 9 07:41:17 2015
@@ -14,7 +14,7 @@ POSTHOOK: query: LOAD DATA LOCAL INPATH
POSTHOOK: type: LOAD
#### A masked pattern was here ####
POSTHOOK: Output: default@myinput1
-Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
@@ -24,7 +24,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@myinput1
#### A masked pattern was here ####
13630578
-Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
PREHOOK: type: QUERY
PREHOOK: Input: default@myinput1
Modified: hive/branches/spark/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/auto_join_without_localtask.q.out Fri Jan 9 07:41:17 2015
@@ -21,11 +21,11 @@ STAGE PLANS:
Stage: Stage-7
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:a
+ $hdt$_1:a
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:a
+ $hdt$_1:a
TableScan
alias: a
Filter Operator
@@ -91,11 +91,11 @@ STAGE PLANS:
Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:a
+ $hdt$_0:a
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:a
+ $hdt$_0:a
TableScan
alias: a
Filter Operator
Modified: hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer3.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer3.q.out Fri Jan 9 07:41:17 2015
@@ -1228,14 +1228,14 @@ STAGE PLANS:
Stage: Stage-9
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:x
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:x
Fetch Operator
limit: -1
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_2:x
+ $hdt$_0:$hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_2:x
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:x
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:x
TableScan
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -1250,7 +1250,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_2:x
+ $hdt$_0:$hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_2:x
TableScan
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer6.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer6.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer6.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer6.q.out Fri Jan 9 07:41:17 2015
@@ -500,14 +500,14 @@ STAGE PLANS:
Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
Fetch Operator
limit: -1
- $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_1:x
+ $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:x
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
TableScan
alias: x
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -522,7 +522,7 @@ STAGE PLANS:
keys:
0 _col0 (type: string)
1 _col0 (type: string)
- $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_1:x
+ $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:x
TableScan
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -3822,11 +3822,11 @@ STAGE PLANS:
Stage: Stage-10
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
+ $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:x
TableScan
alias: x
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
@@ -3894,11 +3894,11 @@ STAGE PLANS:
Stage: Stage-9
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_1:x
+ $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:x
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:$hdt$_1:x
+ $hdt$_0:$hdt$_1:$hdt$_1:$hdt$_1:x
TableScan
alias: x
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer7.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer7.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer7.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/correlationoptimizer7.q.out Fri Jan 9 07:41:17 2015
@@ -29,11 +29,11 @@ STAGE PLANS:
Stage: Stage-9
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -101,11 +101,11 @@ STAGE PLANS:
Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:y
+ $hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:y
+ $hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -213,11 +213,11 @@ STAGE PLANS:
Stage: Stage-6
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -416,11 +416,11 @@ STAGE PLANS:
Stage: Stage-9
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -488,11 +488,11 @@ STAGE PLANS:
Stage: Stage-8
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:y
+ $hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:y
+ $hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
@@ -600,11 +600,11 @@ STAGE PLANS:
Stage: Stage-6
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_0:$hdt$_0:$hdt$_0:$hdt$_1:y
+ $hdt$_0:$hdt$_0:$hdt$_1:y
TableScan
alias: y
Statistics: Num rows: 25 Data size: 191 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/cross_product_check_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/cross_product_check_2.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/cross_product_check_2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/cross_product_check_2.q.out Fri Jan 9 07:41:17 2015
@@ -24,7 +24,7 @@ POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@src
POSTHOOK: Output: database:default
POSTHOOK: Output: default@B
-Warning: Map Join MAPJOIN[12][bigTable=?] in task 'Stage-3:MAPRED' is a cross product
+Warning: Map Join MAPJOIN[11][bigTable=?] in task 'Stage-3:MAPRED' is a cross product
PREHOOK: query: explain select * from A join B
PREHOOK: type: QUERY
POSTHOOK: query: explain select * from A join B
@@ -38,11 +38,11 @@ STAGE PLANS:
Stage: Stage-4
Map Reduce Local Work
Alias -> Map Local Tables:
- $hdt$_0:$hdt$_1:b
+ $hdt$_1:b
Fetch Operator
limit: -1
Alias -> Map Local Operator Tree:
- $hdt$_0:$hdt$_1:b
+ $hdt$_1:b
TableScan
alias: b
Statistics: Num rows: 10 Data size: 104 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/explain_logical.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/explain_logical.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/explain_logical.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/explain_logical.q.out Fri Jan 9 07:41:17 2015
@@ -515,11 +515,11 @@ TOK_QUERY
LOGICAL PLAN:
-$hdt$_0:$hdt$_0:$hdt$_0:srcpart
+$hdt$_0:$hdt$_0:srcpart
TableScan (TS_0)
alias: srcpart
Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
- Filter Operator (FIL_14)
+ Filter Operator (FIL_13)
predicate: key is not null (type: boolean)
Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE
Select Operator (SEL_2)
@@ -543,18 +543,18 @@ $hdt$_0:$hdt$_0:$hdt$_0:srcpart
expressions: _col0 (type: string), _col2 (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
- File Output Operator (FS_13)
+ File Output Operator (FS_12)
compressed: false
Statistics: Num rows: 1100 Data size: 11686 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
-$hdt$_0:$hdt$_1:src2
+$hdt$_1:src2
TableScan (TS_4)
alias: src2
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
- Filter Operator (FIL_15)
+ Filter Operator (FIL_14)
predicate: key is not null (type: boolean)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator (SEL_5)
Modified: hive/branches/spark/ql/src/test/results/clientpositive/groupby_cube1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/groupby_cube1.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/groupby_cube1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/groupby_cube1.q.out Fri Jan 9 07:41:17 2015
@@ -37,11 +37,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -103,6 +103,90 @@ NULL 18 1
NULL 28 1
NULL NULL 6
PREHOOK: query: EXPLAIN
+SELECT key, val, GROUPING__ID, count(1) FROM T1 GROUP BY key, val with cube
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+SELECT key, val, GROUPING__ID, count(1) FROM T1 GROUP BY key, val with cube
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t1
+ Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
+ Select Operator
+ expressions: key (type: string), val (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
+ Group By Operator
+ aggregations: count(1)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: string), _col1 (type: string), _col2 (type: string)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ value expressions: _col3 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ keys: KEY._col0 (type: string), KEY._col1 (type: string), KEY._col2 (type: string)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string), _col2 (type: string), _col3 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT key, val, GROUPING__ID, count(1) FROM T1 GROUP BY key, val with cube
+PREHOOK: type: QUERY
+PREHOOK: Input: default@t1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT key, val, GROUPING__ID, count(1) FROM T1 GROUP BY key, val with cube
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@t1
+#### A masked pattern was here ####
+1 11 3 1
+1 NULL 1 1
+2 12 3 1
+2 NULL 1 1
+3 13 3 1
+3 NULL 1 1
+7 17 3 1
+7 NULL 1 1
+8 18 3 1
+8 28 3 1
+8 NULL 1 2
+NULL 11 2 1
+NULL 12 2 1
+NULL 13 2 1
+NULL 17 2 1
+NULL 18 2 1
+NULL 28 2 1
+NULL NULL 0 6
+PREHOOK: query: EXPLAIN
SELECT key, count(distinct val) FROM T1 GROUP BY key with cube
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN
@@ -193,11 +277,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets2.q.out Fri Jan 9 07:41:17 2015
@@ -36,11 +36,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: string), b (type: string)
- outputColumnNames: a, b
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: count()
- keys: a (type: string), b (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -142,11 +142,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: string), b (type: string), c (type: string)
- outputColumnNames: a, b, c
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: sum(c)
- keys: a (type: string), b (type: string)
+ aggregations: sum(_col2)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -269,12 +269,12 @@ STAGE PLANS:
alias: t2
Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: a (type: string), b (type: string), c (type: int), d (type: int)
- outputColumnNames: a, b, c, d
+ expressions: a (type: string), b (type: string), (c + d) (type: int)
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE
Group By Operator
- aggregations: sum((c + d))
- keys: a (type: string), b (type: string)
+ aggregations: sum(_col2)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 6 Data size: 42 Basic stats: COMPLETE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets3.q.out Fri Jan 9 07:41:17 2015
@@ -55,11 +55,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 72 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: string), b (type: string), c (type: string)
- outputColumnNames: a, b, c
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 72 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: avg(c), count()
- keys: a (type: string), b (type: string), '0' (type: string)
+ aggregations: avg(_col2), count()
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3, _col4
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -142,11 +142,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 72 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: string), b (type: string), c (type: string)
- outputColumnNames: a, b, c
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 72 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: avg(c), count()
- keys: a (type: string), b (type: string)
+ aggregations: avg(_col2), count()
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Modified: hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/groupby_grouping_sets5.q.out Fri Jan 9 07:41:17 2015
@@ -42,43 +42,36 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: string), b (type: string)
- outputColumnNames: a, b
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: a (type: string), b (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- value expressions: _col2 (type: bigint)
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count()
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2, _col3
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
@@ -165,43 +158,36 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: a (type: string), b (type: string)
- outputColumnNames: a, b
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 36 Basic stats: PARTIAL Column stats: NONE
Group By Operator
- aggregations: count(1)
- keys: a (type: string), b (type: string)
+ keys: _col0 (type: string), _col1 (type: string)
mode: hash
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string), _col1 (type: string)
sort order: ++
Map-reduce partition columns: _col0 (type: string), _col1 (type: string)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- value expressions: _col2 (type: bigint)
Reduce Operator Tree:
Group By Operator
- aggregations: count(VALUE._col0)
keys: KEY._col0 (type: string), KEY._col1 (type: string)
mode: mergepartial
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
+ Group By Operator
+ aggregations: count()
+ keys: _col0 (type: string), _col1 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: _col0 (type: string), _col1 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1, _col2
- Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
- File Output Operator
- compressed: false
- table:
- input format: org.apache.hadoop.mapred.SequenceFileInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
Stage: Stage-2
Map Reduce
Modified: hive/branches/spark/ql/src/test/results/clientpositive/groupby_rollup1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/groupby_rollup1.q.out?rev=1650453&r1=1650452&r2=1650453&view=diff
==============================================================================
--- hive/branches/spark/ql/src/test/results/clientpositive/groupby_rollup1.q.out (original)
+++ hive/branches/spark/ql/src/test/results/clientpositive/groupby_rollup1.q.out Fri Jan 9 07:41:17 2015
@@ -37,11 +37,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
@@ -187,11 +187,11 @@ STAGE PLANS:
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Select Operator
expressions: key (type: string), val (type: string)
- outputColumnNames: key, val
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 0 Data size: 30 Basic stats: PARTIAL Column stats: NONE
Group By Operator
aggregations: count(1)
- keys: key (type: string), val (type: string), '0' (type: string)
+ keys: _col0 (type: string), _col1 (type: string), '0' (type: string)
mode: hash
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE