You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2015/04/07 23:51:53 UTC
svn commit: r1671959 [4/5] - in /hive/branches/cbo: ./
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/
hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/
hcatalog/hcatalog-pig-adapter/src/test/ja...
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java Tue Apr 7 21:51:52 2015
@@ -517,15 +517,10 @@ public final class ConstantPropagateProc
if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(clz)) {
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
TypeInfo typeInfo = poi.getTypeInfo();
- // Handling parameterized types (varchar etc).
- if (typeInfo.getTypeName().contains(serdeConstants.VARCHAR_TYPE_NAME)
- || typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) {
-
- // Do not support parameterized types.
- return null;
- }
o = poi.getPrimitiveJavaObject(o);
- if (typeInfo.getTypeName().contains(serdeConstants.DECIMAL_TYPE_NAME)) {
+ if (typeInfo.getTypeName().contains(serdeConstants.DECIMAL_TYPE_NAME) ||
+ typeInfo.getTypeName().contains(serdeConstants.VARCHAR_TYPE_NAME) ||
+ typeInfo.getTypeName().contains(serdeConstants.CHAR_TYPE_NAME)) {
return new ExprNodeConstantDesc(typeInfo, o);
}
} else if (PrimitiveObjectInspectorUtils.isPrimitiveJavaClass(clz)) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Tue Apr 7 21:51:52 2015
@@ -163,7 +163,7 @@ public class ConvertJoinMapJoin implemen
// map join operator by default has no bucket cols and num of reduce sinks
// reduced by 1
mapJoinOp
- .setOpTraits(new OpTraits(null, -1, null, joinOp.getOpTraits().getNumReduceSinks()));
+.setOpTraits(new OpTraits(null, -1, null));
mapJoinOp.setStatistics(joinOp.getStatistics());
// propagate this change till the next RS
for (Operator<? extends OperatorDesc> childOp : mapJoinOp.getChildOperators()) {
@@ -178,8 +178,7 @@ public class ConvertJoinMapJoin implemen
TezBucketJoinProcCtx tezBucketJoinProcCtx) throws SemanticException {
// we cannot convert to bucket map join, we cannot convert to
// map join either based on the size. Check if we can convert to SMB join.
- if ((context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false)
- || (joinOp.getOpTraits().getNumReduceSinks() >= 2)) {
+ if (context.conf.getBoolVar(HiveConf.ConfVars.HIVE_AUTO_SORTMERGE_JOIN) == false) {
convertJoinSMBJoin(joinOp, context, 0, 0, false);
return null;
}
@@ -254,9 +253,9 @@ public class ConvertJoinMapJoin implemen
CommonMergeJoinOperator mergeJoinOp =
(CommonMergeJoinOperator) OperatorFactory.get(new CommonMergeJoinDesc(numBuckets,
mapJoinConversionPos, mapJoinDesc), joinOp.getSchema());
- int numReduceSinks = joinOp.getOpTraits().getNumReduceSinks();
- OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp
- .getOpTraits().getSortCols(), numReduceSinks);
+ OpTraits opTraits =
+ new OpTraits(joinOp.getOpTraits().getBucketColNames(), numBuckets, joinOp.getOpTraits()
+ .getSortCols());
mergeJoinOp.setOpTraits(opTraits);
mergeJoinOp.setStatistics(joinOp.getStatistics());
@@ -321,8 +320,7 @@ public class ConvertJoinMapJoin implemen
if (currentOp instanceof ReduceSinkOperator) {
return;
}
- currentOp.setOpTraits(new OpTraits(null, -1, null,
- currentOp.getOpTraits().getNumReduceSinks()));
+ currentOp.setOpTraits(new OpTraits(null, -1, null));
for (Operator<? extends OperatorDesc> childOp : currentOp.getChildOperators()) {
if ((childOp instanceof ReduceSinkOperator) || (childOp instanceof GroupByOperator)) {
break;
@@ -345,7 +343,7 @@ public class ConvertJoinMapJoin implemen
// we can set the traits for this join operator
OpTraits opTraits = new OpTraits(joinOp.getOpTraits().getBucketColNames(),
- tezBucketJoinProcCtx.getNumBuckets(), null, joinOp.getOpTraits().getNumReduceSinks());
+ tezBucketJoinProcCtx.getNumBuckets(), null);
mapJoinOp.setOpTraits(opTraits);
mapJoinOp.setStatistics(joinOp.getStatistics());
setNumberOfBucketsOnChildren(mapJoinOp);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MergeJoinProc.java Tue Apr 7 21:51:52 2015
@@ -70,12 +70,11 @@ public class MergeJoinProc implements No
context.opMergeJoinWorkMap.put(mergeJoinOp, mergeWork);
}
+ mergeWork.addMergedWork(null, parentWork, context.leafOperatorToFollowingWork);
mergeWork.setMergeJoinOperator(mergeJoinOp);
- mergeWork.addMergedWork(null, parentWork);
tezWork.setVertexType(mergeWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
for (BaseWork grandParentWork : tezWork.getParents(parentWork)) {
- parentWork.setName(grandParentWork.getName());
TezEdgeProperty edgeProp = tezWork.getEdgeProperty(grandParentWork, parentWork);
tezWork.disconnect(grandParentWork, parentWork);
tezWork.connect(grandParentWork, mergeWork, edgeProp);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkMapJoinProc.java Tue Apr 7 21:51:52 2015
@@ -179,13 +179,44 @@ public class ReduceSinkMapJoinProc imple
parentRS.getConf().setReducerTraits(EnumSet.of(FIXED));
numBuckets = (Integer) joinConf.getBigTableBucketNumMapping().values().toArray()[0];
- Operator<?> rootOp = OperatorUtils.findSingleOperatorUpstream(mapJoinOp.getParentOperators()
- .get(joinConf.getPosBigTable()), TableScanOperator.class);
-
- if (rootOp instanceof TableScanOperator) { // we will run in mapper
- edgeType = EdgeType.CUSTOM_EDGE;
- } else { // we will run in reducer
- edgeType = EdgeType.CUSTOM_SIMPLE_EDGE;
+ /*
+ * Here, we can be in one of 4 states.
+ *
+ * 1. If map join work is null implies that we have not yet traversed the big table side. We
+ * just need to see if we can find a reduce sink operator in the big table side. This would
+ * imply a reduce side operation.
+ *
+ * 2. If we don't find a reducesink in 1 it has to be the case that it is a map side operation.
+ *
+ * 3. If we have already created a work item for the big table side, we need to see if we can
+ * find a table scan operator in the big table side. This would imply a map side operation.
+ *
+ * 4. If we don't find a table scan operator, it has to be a reduce side operation.
+ */
+ if (mapJoinWork == null) {
+ Operator<?> rootOp =
+ OperatorUtils.findSingleOperatorUpstream(
+ mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()),
+ ReduceSinkOperator.class);
+ if (rootOp == null) {
+ // likely we found a table scan operator
+ edgeType = EdgeType.CUSTOM_EDGE;
+ } else {
+ // we have found a reduce sink
+ edgeType = EdgeType.CUSTOM_SIMPLE_EDGE;
+ }
+ } else {
+ Operator<?> rootOp =
+ OperatorUtils.findSingleOperatorUpstream(
+ mapJoinOp.getParentOperators().get(joinConf.getPosBigTable()),
+ TableScanOperator.class);
+ if (rootOp != null) {
+ // likely we found a table scan operator
+ edgeType = EdgeType.CUSTOM_EDGE;
+ } else {
+ // we have found a reduce sink
+ edgeType = EdgeType.CUSTOM_SIMPLE_EDGE;
+ }
}
}
TezEdgeProperty edgeProp = new TezEdgeProperty(null, edgeType, numBuckets);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java Tue Apr 7 21:51:52 2015
@@ -50,19 +50,19 @@ import org.apache.hadoop.hive.ql.plan.Op
* 1. Bucketing columns.
* 2. Table
* 3. Pruned partitions
- *
+ *
* Bucketing columns refer to not to the bucketing columns from the table object but instead
* to the dynamic 'bucketing' done by operators such as reduce sinks and group-bys.
* All the operators have a translation from their input names to the output names corresponding
* to the bucketing column. The colExprMap that is a part of every operator is used in this
* transformation.
- *
+ *
* The table object is used for the base-case in map-reduce when deciding to perform a bucket
* map join. This object is used in the BucketMapJoinProc to find if number of files for the
* table correspond to the number of buckets specified in the meta data.
- *
+ *
* The pruned partition information has the same purpose as the table object at the moment.
- *
+ *
* The traits of sorted-ness etc. can be populated as well for future optimizations to make use of.
*/
@@ -106,13 +106,11 @@ public class OpTraitsRulesProcFactory {
List<List<String>> listBucketCols = new ArrayList<List<String>>();
listBucketCols.add(bucketCols);
int numBuckets = -1;
- int numReduceSinks = 1;
OpTraits parentOpTraits = rs.getParentOperators().get(0).getConf().getTraits();
if (parentOpTraits != null) {
numBuckets = parentOpTraits.getNumBuckets();
- numReduceSinks += parentOpTraits.getNumReduceSinks();
}
- OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols, numReduceSinks);
+ OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listBucketCols);
rs.setOpTraits(opTraits);
return null;
}
@@ -132,8 +130,8 @@ public class OpTraitsRulesProcFactory {
// construct a mapping of (Partition->bucket file names) and (Partition -> bucket number)
if (!partitions.isEmpty()) {
for (Partition p : partitions) {
- List<String> fileNames =
- AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(),
+ List<String> fileNames =
+ AbstractBucketJoinProc.getBucketFilePathsOfPartition(p.getDataLocation(),
pGraphContext);
// The number of files for the table should be same as number of
// buckets.
@@ -146,8 +144,8 @@ public class OpTraitsRulesProcFactory {
}
} else {
- List<String> fileNames =
- AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(),
+ List<String> fileNames =
+ AbstractBucketJoinProc.getBucketFilePathsOfPartition(tbl.getDataLocation(),
pGraphContext);
Integer num = new Integer(tbl.getNumBuckets());
@@ -188,7 +186,7 @@ public class OpTraitsRulesProcFactory {
sortedColsList.add(sortCols);
}
// num reduce sinks hardcoded to 0 because TS has no parents
- OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList, 0);
+ OpTraits opTraits = new OpTraits(bucketColsList, numBuckets, sortedColsList);
ts.setOpTraits(opTraits);
return null;
}
@@ -213,13 +211,8 @@ public class OpTraitsRulesProcFactory {
}
List<List<String>> listBucketCols = new ArrayList<List<String>>();
- int numReduceSinks = 0;
- OpTraits parentOpTraits = gbyOp.getParentOperators().get(0).getOpTraits();
- if (parentOpTraits != null) {
- numReduceSinks = parentOpTraits.getNumReduceSinks();
- }
listBucketCols.add(gbyKeys);
- OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols, numReduceSinks);
+ OpTraits opTraits = new OpTraits(listBucketCols, -1, listBucketCols);
gbyOp.setOpTraits(opTraits);
return null;
}
@@ -255,7 +248,7 @@ public class OpTraitsRulesProcFactory {
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
SelectOperator selOp = (SelectOperator) nd;
- List<List<String>> parentBucketColNames =
+ List<List<String>> parentBucketColNames =
selOp.getParentOperators().get(0).getOpTraits().getBucketColNames();
List<List<String>> listBucketCols = null;
@@ -264,7 +257,7 @@ public class OpTraitsRulesProcFactory {
if (parentBucketColNames != null) {
listBucketCols = getConvertedColNames(parentBucketColNames, selOp);
}
- List<List<String>> parentSortColNames =
+ List<List<String>> parentSortColNames =
selOp.getParentOperators().get(0).getOpTraits().getSortCols();
if (parentSortColNames != null) {
listSortCols = getConvertedColNames(parentSortColNames, selOp);
@@ -272,13 +265,11 @@ public class OpTraitsRulesProcFactory {
}
int numBuckets = -1;
- int numReduceSinks = 0;
OpTraits parentOpTraits = selOp.getParentOperators().get(0).getOpTraits();
if (parentOpTraits != null) {
numBuckets = parentOpTraits.getNumBuckets();
- numReduceSinks = parentOpTraits.getNumReduceSinks();
}
- OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols, numReduceSinks);
+ OpTraits opTraits = new OpTraits(listBucketCols, numBuckets, listSortCols);
selOp.setOpTraits(opTraits);
return null;
}
@@ -307,13 +298,10 @@ public class OpTraitsRulesProcFactory {
OpTraits parentOpTraits = rsOp.getOpTraits();
bucketColsList.add(getOutputColNames(joinOp, parentOpTraits.getBucketColNames(), pos));
sortColsList.add(getOutputColNames(joinOp, parentOpTraits.getSortCols(), pos));
- if (parentOpTraits.getNumReduceSinks() > numReduceSinks) {
- numReduceSinks = parentOpTraits.getNumReduceSinks();
- }
pos++;
}
- joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList, numReduceSinks));
+ joinOp.setOpTraits(new OpTraits(bucketColsList, -1, bucketColsList));
return null;
}
@@ -366,17 +354,7 @@ public class OpTraitsRulesProcFactory {
Object... nodeOutputs) throws SemanticException {
@SuppressWarnings("unchecked")
Operator<? extends OperatorDesc> operator = (Operator<? extends OperatorDesc>) nd;
-
- int numReduceSinks = 0;
- for (Operator<?> parentOp : operator.getParentOperators()) {
- if (parentOp.getOpTraits() == null) {
- continue;
- }
- if (parentOp.getOpTraits().getNumReduceSinks() > numReduceSinks) {
- numReduceSinks = parentOp.getOpTraits().getNumReduceSinks();
- }
- }
- OpTraits opTraits = new OpTraits(null, -1, null, numReduceSinks);
+ OpTraits opTraits = new OpTraits(null, -1, null);
operator.setOpTraits(opTraits);
return null;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java Tue Apr 7 21:51:52 2015
@@ -103,7 +103,7 @@ public class SparkMapJoinOptimizer imple
}
// we can set the traits for this join operator
- OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null, joinOp.getOpTraits().getNumReduceSinks());
+ OpTraits opTraits = new OpTraits(bucketColNames, numBuckets, null);
mapJoinOp.setOpTraits(opTraits);
mapJoinOp.setStatistics(joinOp.getStatistics());
setNumberOfBucketsOnChildren(mapJoinOp);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java Tue Apr 7 21:51:52 2015
@@ -105,7 +105,7 @@ public class GenTezUtils {
float minPartitionFactor = context.conf.getFloatVar(HiveConf.ConfVars.TEZ_MIN_PARTITION_FACTOR);
long bytesPerReducer = context.conf.getLongVar(HiveConf.ConfVars.BYTESPERREDUCER);
- ReduceWork reduceWork = new ReduceWork("Reducer "+ (++sequenceNumber));
+ ReduceWork reduceWork = new ReduceWork(Utilities.REDUCENAME + (++sequenceNumber));
LOG.debug("Adding reduce work (" + reduceWork.getName() + ") for " + root);
reduceWork.setReducer(root);
reduceWork.setNeedsTagging(GenMapRedUtils.needsTagging(reduceWork));
@@ -180,7 +180,7 @@ public class GenTezUtils {
public MapWork createMapWork(GenTezProcContext context, Operator<?> root,
TezWork tezWork, PrunedPartitionList partitions) throws SemanticException {
assert root.getParentOperators().isEmpty();
- MapWork mapWork = new MapWork("Map "+ (++sequenceNumber));
+ MapWork mapWork = new MapWork(Utilities.MAPNAME + (++sequenceNumber));
LOG.debug("Adding map work (" + mapWork.getName() + ") for " + root);
// map work starts with table scan operators
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezWork.java Tue Apr 7 21:51:52 2015
@@ -166,7 +166,7 @@ public class GenTezWork implements NodeP
}
// connect the work correctly.
work.addSortCols(root.getOpTraits().getSortCols().get(0));
- mergeJoinWork.addMergedWork(work, null);
+ mergeJoinWork.addMergedWork(work, null, context.leafOperatorToFollowingWork);
Operator<? extends OperatorDesc> parentOp =
getParentFromStack(context.currentMergeJoinOperator, stack);
int pos = context.currentMergeJoinOperator.getTagForOperator(parentOp);
@@ -268,6 +268,7 @@ public class GenTezWork implements NodeP
if (LOG.isDebugEnabled()) {
LOG.debug("Removing " + parent + " as parent from " + root);
}
+ context.leafOperatorToFollowingWork.remove(parent);
context.leafOperatorToFollowingWork.put(parent, work);
root.removeParent(parent);
}
@@ -326,7 +327,7 @@ public class GenTezWork implements NodeP
MergeJoinWork mergeJoinWork = (MergeJoinWork) followingWork;
CommonMergeJoinOperator mergeJoinOp = mergeJoinWork.getMergeJoinOperator();
work.setTag(mergeJoinOp.getTagForOperator(operator));
- mergeJoinWork.addMergedWork(null, work);
+ mergeJoinWork.addMergedWork(null, work, context.leafOperatorToFollowingWork);
tezWork.setVertexType(mergeJoinWork, VertexType.MULTI_INPUT_UNINITIALIZED_EDGES);
for (BaseWork parentWork : tezWork.getParents(work)) {
TezEdgeProperty edgeProp = tezWork.getEdgeProperty(parentWork, work);
@@ -399,7 +400,7 @@ public class GenTezWork implements NodeP
return null;
}
- private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperator rs)
+ private int getFollowingWorkIndex(TezWork tezWork, UnionWork unionWork, ReduceSinkOperator rs)
throws SemanticException {
int index = 0;
for (BaseWork baseWork : tezWork.getChildren(unionWork)) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Apr 7 21:51:52 2015
@@ -1577,13 +1577,16 @@ public class SemanticAnalyzer extends Ba
// Disallow INSERT INTO on bucketized tables
boolean isAcid = isAcidTable(tab);
- if (qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName()) &&
+ boolean isTableWrittenTo = qb.getParseInfo().isInsertIntoTable(tab.getDbName(), tab.getTableName());
+ if (isTableWrittenTo &&
tab.getNumBuckets() > 0 && !isAcid) {
throw new SemanticException(ErrorMsg.INSERT_INTO_BUCKETIZED_TABLE.
getMsg("Table: " + tab_name));
}
// Disallow update and delete on non-acid tables
- if ((updating() || deleting()) && !isAcid) {
+ if ((updating() || deleting()) && !isAcid && isTableWrittenTo) {
+ //isTableWrittenTo: delete from acidTbl where a in (select id from nonAcidTable)
+ //so only assert this if we are actually writing to this table
// isAcidTable above also checks for whether we are using an acid compliant
// transaction manager. But that has already been caught in
// UpdateDeleteSemanticAnalyzer, so if we are updating or deleting and getting nonAcid
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java Tue Apr 7 21:51:52 2015
@@ -33,7 +33,7 @@ public class AbstractOperatorDesc implem
static {
PTFUtils.makeTransient(AbstractOperatorDesc.class, "opProps");
}
-
+
@Override
@Explain(skipHeader = true, displayName = "Statistics")
public Statistics getStatistics() {
@@ -50,14 +50,18 @@ public class AbstractOperatorDesc implem
throw new CloneNotSupportedException("clone not supported");
}
+ public boolean getVectorMode() {
+ return vectorMode;
+ }
+
public void setVectorMode(boolean vm) {
this.vectorMode = vm;
}
-
+
public OpTraits getTraits() {
return opTraits;
}
-
+
public void setTraits(OpTraits opTraits) {
this.opTraits = opTraits;
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java Tue Apr 7 21:51:52 2015
@@ -34,7 +34,7 @@ import org.apache.hadoop.mapred.JobConf;
* BaseWork. Base class for any "work" that's being done on the cluster. Items like stats
* gathering that are commonly used regardless of the type of work live here.
*/
-@SuppressWarnings({"serial", "deprecation"})
+@SuppressWarnings({"serial"})
public abstract class BaseWork extends AbstractOperatorDesc {
// dummyOps is a reference to all the HashTableDummy operators in the
@@ -60,7 +60,6 @@ public abstract class BaseWork extends A
// Vectorization.
protected Map<String, Map<Integer, String>> allScratchColumnVectorTypeMaps = null;
protected Map<String, Map<String, Integer>> allColumnVectorMaps = null;
- protected boolean vectorMode = false;
public void setGatheringStats(boolean gatherStats) {
this.gatheringStats = gatherStats;
@@ -175,15 +174,6 @@ public abstract class BaseWork extends A
this.mrLocalWork = mapLocalWork;
}
- @Override
- public void setVectorMode(boolean vectorMode) {
- this.vectorMode = vectorMode;
- }
-
- public boolean getVectorMode() {
- return vectorMode;
- }
-
public abstract void configureJobConf(JobConf job);
public void setTag(int tag) {
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java Tue Apr 7 21:51:52 2015
@@ -21,11 +21,13 @@ package org.apache.hadoop.hive.ql.plan;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.mapred.JobConf;
public class MergeJoinWork extends BaseWork {
@@ -65,7 +67,8 @@ public class MergeJoinWork extends BaseW
this.mergeJoinOp = mergeJoinOp;
}
- public void addMergedWork(BaseWork work, BaseWork connectWork) {
+ public void addMergedWork(BaseWork work, BaseWork connectWork,
+ Map<Operator<?>, BaseWork> leafOperatorToFollowingWork) {
if (work != null) {
if ((bigTableWork != null) && (bigTableWork != work)) {
assert false;
@@ -76,6 +79,39 @@ public class MergeJoinWork extends BaseW
if (connectWork != null) {
this.mergeWorkList.add(connectWork);
+ if ((connectWork instanceof ReduceWork) && (bigTableWork != null)) {
+ /*
+ * For tez to route data from an up-stream vertex correctly to the following vertex, the
+ * output name in the reduce sink needs to be setup appropriately. In the case of reduce
+ * side merge work, we need to ensure that the parent work that provides data to this merge
+ * work is setup to point to the right vertex name - the main work name.
+ *
+ * In this case, if the big table work has already been created, we can hook up the merge
+ * work items for the small table correctly.
+ */
+ setReduceSinkOutputName(connectWork, leafOperatorToFollowingWork, bigTableWork.getName());
+ }
+ }
+
+ if (work != null) {
+ /*
+ * Same reason as above. This is the case when we have the main work item after the merge work
+ * has been created for the small table side.
+ */
+ for (BaseWork mergeWork : mergeWorkList) {
+ if (mergeWork instanceof ReduceWork) {
+ setReduceSinkOutputName(mergeWork, leafOperatorToFollowingWork, work.getName());
+ }
+ }
+ }
+ }
+
+ private void setReduceSinkOutputName(BaseWork mergeWork,
+ Map<Operator<?>, BaseWork> leafOperatorToFollowingWork, String name) {
+ for (Entry<Operator<?>, BaseWork> entry : leafOperatorToFollowingWork.entrySet()) {
+ if (entry.getValue() == mergeWork) {
+ ((ReduceSinkOperator) entry.getKey()).getConf().setOutputName(name);
+ }
}
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/OpTraits.java Tue Apr 7 21:51:52 2015
@@ -25,14 +25,11 @@ public class OpTraits {
List<List<String>> bucketColNames;
List<List<String>> sortColNames;
int numBuckets;
- int numReduceSinks;
- public OpTraits(List<List<String>> bucketColNames, int numBuckets,
- List<List<String>> sortColNames, int numReduceSinks) {
+ public OpTraits(List<List<String>> bucketColNames, int numBuckets, List<List<String>> sortColNames) {
this.bucketColNames = bucketColNames;
this.numBuckets = numBuckets;
this.sortColNames = sortColNames;
- this.numReduceSinks = numReduceSinks;
}
public List<List<String>> getBucketColNames() {
@@ -58,12 +55,4 @@ public class OpTraits {
public List<List<String>> getSortCols() {
return sortColNames;
}
-
- public void setNumReduceSinks(int numReduceSinks) {
- this.numReduceSinks = numReduceSinks;
- }
-
- public int getNumReduceSinks() {
- return this.numReduceSinks;
- }
}
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java Tue Apr 7 21:51:52 2015
@@ -361,19 +361,22 @@ public class StorageBasedAuthorizationPr
final FileSystem fs = path.getFileSystem(conf);
- if (fs.exists(path)) {
- checkPermissions(fs, path, actions, authenticator.getUserName());
+ FileStatus pathStatus = FileUtils.getFileStatusOrNull(fs, path);
+ if (pathStatus != null) {
+ checkPermissions(fs, pathStatus, actions, authenticator.getUserName());
} else if (path.getParent() != null) {
// find the ancestor which exists to check its permissions
Path par = path.getParent();
+ FileStatus parStatus = null;
while (par != null) {
- if (fs.exists(par)) {
+ parStatus = FileUtils.getFileStatusOrNull(fs, par);
+ if (parStatus != null) {
break;
}
par = par.getParent();
}
- checkPermissions(fs, par, actions, authenticator.getUserName());
+ checkPermissions(fs, parStatus, actions, authenticator.getUserName());
}
}
@@ -382,18 +385,20 @@ public class StorageBasedAuthorizationPr
* does not exists, it returns.
*/
@SuppressWarnings("deprecation")
- protected static void checkPermissions(final FileSystem fs, final Path path,
+ protected static void checkPermissions(final FileSystem fs, final FileStatus stat,
final EnumSet<FsAction> actions, String user) throws IOException,
AccessControlException, HiveException {
- try {
- FileStatus stat = fs.getFileStatus(path);
- for (FsAction action : actions) {
- FileUtils.checkFileAccessWithImpersonation(fs, stat, action, user);
- }
- } catch (FileNotFoundException fnfe) {
+ if (stat == null) {
// File named by path doesn't exist; nothing to validate.
return;
+ }
+ FsAction checkActions = FsAction.NONE;
+ for (FsAction action : actions) {
+ checkActions = checkActions.or(action);
+ }
+ try {
+ FileUtils.checkFileAccessWithImpersonation(fs, stat, checkActions, user);
} catch (org.apache.hadoop.fs.permission.AccessControlException ace) {
// Older hadoop version will throw this @deprecated Exception.
throw accessControlException(ace);
Modified: hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java (original)
+++ hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java Tue Apr 7 21:51:52 2015
@@ -385,8 +385,7 @@ public class SQLAuthorizationUtils {
FileSystem fs;
try {
fs = FileSystem.get(filePath.toUri(), conf);
- Path path = FileUtils.getPathOrParentThatExists(fs, filePath);
- FileStatus fileStatus = fs.getFileStatus(path);
+ FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath);
if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) {
availPrivs.addPrivilege(SQLPrivTypeGrant.OWNER_PRIV);
}
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java Tue Apr 7 21:51:52 2015
@@ -66,6 +66,7 @@ import org.apache.hadoop.hive.ql.io.Comb
import org.apache.hadoop.hive.ql.io.HiveInputFormat;
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.InputFormatChecker;
+import org.apache.hadoop.hive.ql.io.orc.OrcInputFormat.SplitStrategy;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -98,7 +99,6 @@ import org.apache.hadoop.mapred.OutputFo
import org.apache.hadoop.mapred.RecordWriter;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -394,22 +394,9 @@ public class TestInputOutputFormat {
OrcInputFormat.getInputPaths(conf));
}
- static class TestContext extends OrcInputFormat.Context {
- List<Runnable> queue = new ArrayList<Runnable>();
-
- TestContext(Configuration conf) {
- super(conf);
- }
-
- @Override
- public void schedule(Runnable runnable) {
- queue.add(runnable);
- }
- }
-
@Test
public void testFileGenerator() throws Exception {
- TestContext context = new TestContext(conf);
+ OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
MockFileSystem fs = new MockFileSystem(conf,
new MockFile("mock:/a/b/part-00", 1000, new byte[0]),
new MockFile("mock:/a/b/part-01", 1000, new byte[0]),
@@ -419,21 +406,22 @@ public class TestInputOutputFormat {
OrcInputFormat.FileGenerator gen =
new OrcInputFormat.FileGenerator(context, fs,
new MockPath(fs, "mock:/a/b"));
- gen.run();
- if (context.getErrors().size() > 0) {
- for(Throwable th: context.getErrors()) {
- System.out.println(StringUtils.stringifyException(th));
- }
- throw new IOException("Errors during file generation");
- }
- assertEquals(-1, context.getSchedulers());
- assertEquals(3, context.queue.size());
- assertEquals(new Path("mock:/a/b/part-00"),
- ((OrcInputFormat.SplitGenerator) context.queue.get(0)).getPath());
- assertEquals(new Path("mock:/a/b/part-01"),
- ((OrcInputFormat.SplitGenerator) context.queue.get(1)).getPath());
- assertEquals(new Path("mock:/a/b/part-04"),
- ((OrcInputFormat.SplitGenerator) context.queue.get(2)).getPath());
+ SplitStrategy splitStrategy = gen.call();
+ assertEquals(true, splitStrategy instanceof OrcInputFormat.BISplitStrategy);
+
+ conf.set("mapreduce.input.fileinputformat.split.maxsize", "500");
+ context = new OrcInputFormat.Context(conf);
+ fs = new MockFileSystem(conf,
+ new MockFile("mock:/a/b/part-00", 1000, new byte[1000]),
+ new MockFile("mock:/a/b/part-01", 1000, new byte[1000]),
+ new MockFile("mock:/a/b/_part-02", 1000, new byte[1000]),
+ new MockFile("mock:/a/b/.part-03", 1000, new byte[1000]),
+ new MockFile("mock:/a/b/part-04", 1000, new byte[1000]));
+ gen = new OrcInputFormat.FileGenerator(context, fs,
+ new MockPath(fs, "mock:/a/b"));
+ splitStrategy = gen.call();
+ assertEquals(true, splitStrategy instanceof OrcInputFormat.ETLSplitStrategy);
+
}
public static class MockBlock {
@@ -848,11 +836,10 @@ public class TestInputOutputFormat {
new MockBlock("host5-1", "host5-2", "host5-3")));
OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
OrcInputFormat.SplitGenerator splitter =
- new OrcInputFormat.SplitGenerator(context, fs,
+ new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
fs.getFileStatus(new Path("/a/file")), null, true,
- new ArrayList<Long>(), true);
- splitter.createSplit(0, 200, null);
- OrcSplit result = context.getResult(-1);
+ new ArrayList<Long>(), true, null, null));
+ OrcSplit result = splitter.createSplit(0, 200, null);
assertEquals(0, result.getStart());
assertEquals(200, result.getLength());
assertEquals("mock:/a/file", result.getPath().toString());
@@ -861,15 +848,13 @@ public class TestInputOutputFormat {
assertEquals("host1-1", locs[0]);
assertEquals("host1-2", locs[1]);
assertEquals("host1-3", locs[2]);
- splitter.createSplit(500, 600, null);
- result = context.getResult(-1);
+ result = splitter.createSplit(500, 600, null);
locs = result.getLocations();
assertEquals(3, locs.length);
assertEquals("host2-1", locs[0]);
assertEquals("host0", locs[1]);
assertEquals("host2-3", locs[2]);
- splitter.createSplit(0, 2500, null);
- result = context.getResult(-1);
+ result = splitter.createSplit(0, 2500, null);
locs = result.getLocations();
assertEquals(1, locs.length);
assertEquals("host0", locs[0]);
@@ -892,48 +877,36 @@ public class TestInputOutputFormat {
conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 200);
OrcInputFormat.Context context = new OrcInputFormat.Context(conf);
OrcInputFormat.SplitGenerator splitter =
- new OrcInputFormat.SplitGenerator(context, fs,
+ new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
fs.getFileStatus(new Path("/a/file")), null, true,
- new ArrayList<Long>(), true);
- splitter.run();
- if (context.getErrors().size() > 0) {
- for(Throwable th: context.getErrors()) {
- System.out.println(StringUtils.stringifyException(th));
- }
- throw new IOException("Errors during splitting");
- }
- OrcSplit result = context.getResult(0);
+ new ArrayList<Long>(), true, null, null));
+ List<OrcSplit> results = splitter.call();
+ OrcSplit result = results.get(0);
assertEquals(3, result.getStart());
assertEquals(497, result.getLength());
- result = context.getResult(1);
+ result = results.get(1);
assertEquals(500, result.getStart());
assertEquals(600, result.getLength());
- result = context.getResult(2);
+ result = results.get(2);
assertEquals(1100, result.getStart());
assertEquals(400, result.getLength());
- result = context.getResult(3);
+ result = results.get(3);
assertEquals(1500, result.getStart());
assertEquals(300, result.getLength());
- result = context.getResult(4);
+ result = results.get(4);
assertEquals(1800, result.getStart());
assertEquals(200, result.getLength());
// test min = 0, max = 0 generates each stripe
conf.setInt(OrcInputFormat.MIN_SPLIT_SIZE, 0);
conf.setInt(OrcInputFormat.MAX_SPLIT_SIZE, 0);
context = new OrcInputFormat.Context(conf);
- splitter = new OrcInputFormat.SplitGenerator(context, fs,
+ splitter = new OrcInputFormat.SplitGenerator(new OrcInputFormat.SplitInfo(context, fs,
fs.getFileStatus(new Path("/a/file")), null, true, new ArrayList<Long>(),
- true);
- splitter.run();
- if (context.getErrors().size() > 0) {
- for(Throwable th: context.getErrors()) {
- System.out.println(StringUtils.stringifyException(th));
- }
- throw new IOException("Errors during splitting");
- }
+ true, null, null));
+ results = splitter.call();
for(int i=0; i < stripeSizes.length; ++i) {
assertEquals("checking stripe " + i + " size",
- stripeSizes[i], context.getResult(i).getLength());
+ stripeSizes[i], results.get(i).getLength());
}
}
Modified: hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java (original)
+++ hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHive.java Tue Apr 7 21:51:52 2015
@@ -704,7 +704,7 @@ public class TestHive extends TestCase {
index.getIndexName());
assertEquals("Table names don't match for index: " + indexName, tableName,
index.getOrigTableName());
- assertEquals("Index table names didn't match for index: " + indexName, qIndexTableName,
+ assertEquals("Index table names didn't match for index: " + indexName, indexTableName,
index.getIndexTableName());
assertEquals("Index handler classes didn't match for index: " + indexName,
indexHandlerClass, index.getIndexHandlerClass());
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/groupby3_map.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/groupby3_map.q?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/groupby3_map.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/groupby3_map.q Tue Apr 7 21:51:52 2015
@@ -29,6 +29,16 @@ INSERT OVERWRITE TABLE dest1 SELECT
variance(substr(src.value,5)),
var_samp(substr(src.value,5));
-SELECT dest1.* FROM dest1;
+SELECT
+c1,
+c2,
+round(c3, 11) c3,
+c4,
+c5,
+round(c6, 11) c6,
+round(c7, 11) c7,
+round(c8, 5) c8,
+round(c9, 9) c9
+FROM dest1;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/tez_join.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/tez_join.q?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/tez_join.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/tez_join.q Tue Apr 7 21:51:52 2015
@@ -24,20 +24,3 @@ join
(select rt2.id from
(select t2.id, t2.od from t2 order by t2.id, t2.od) rt2) vt2
where vt1.id=vt2.id;
-
-explain
-select vt1.id from
-(select rt1.id from
-(select t1.id, t1.od, count(*) from t1 group by t1.id, t1.od) rt1) vt1
-join
-(select rt2.id from
-(select t2.id, t2.od, count(*) from t2 group by t2.id, t2.od) rt2) vt2
-where vt1.id=vt2.id;
-
-select vt1.id from
-(select rt1.id from
-(select t1.id, t1.od, count(*) from t1 group by t1.id, t1.od) rt1) vt1
-join
-(select rt2.id from
-(select t2.id, t2.od, count(*) from t2 group by t2.id, t2.od) rt2) vt2
-where vt1.id=vt2.id;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_1.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_1.q?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_1.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_1.q Tue Apr 7 21:51:52 2015
@@ -33,3 +33,21 @@ set hive.auto.convert.join.noconditional
explain
select count(*) from tab s1 join tab s3 on s1.key=s3.key;
+set hive.auto.convert.join=false;
+
+explain
+select count(*) from
+(select rt1.id from
+(select t1.key as id, t1.value as od from tab t1 order by id, od) rt1) vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.id=vt2.id;
+
+select count(*) from
+(select rt1.id from
+(select t1.key as id, t1.value as od from tab t1 order by id, od) rt1) vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.id=vt2.id;
Modified: hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_main.q
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_main.q?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_main.q (original)
+++ hive/branches/cbo/ql/src/test/queries/clientpositive/tez_smb_main.q Tue Apr 7 21:51:52 2015
@@ -82,3 +82,20 @@ UNION ALL
select s2.key as key, s2.value as value from tab s2
) a join tab_part b on (a.key = b.key);
+explain
+select count(*) from
+(select rt1.id from
+(select t1.key as id, t1.value as od from tab t1 order by id, od) rt1) vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.id=vt2.id;
+
+select count(*) from
+(select rt1.id from
+(select t1.key as id, t1.value as od from tab t1 order by id, od) rt1) vt1
+join
+(select rt2.id from
+(select t2.key as id, t2.value as od from tab_part t2 order by id, od) rt2) vt2
+where vt1.id=vt2.id;
+
Modified: hive/branches/cbo/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientnegative/alter_concatenate_indexed_table.q.out Tue Apr 7 21:51:52 2015
@@ -76,5 +76,5 @@ PREHOOK: query: show indexes on src_rc_c
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: show indexes on src_rc_concatenate_test
POSTHOOK: type: SHOWINDEXES
-src_rc_concatenate_test_index src_rc_concatenate_test key default.default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact
+src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact
FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: can not do merge because source table default.src_rc_concatenate_test is indexed.
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/alter_concatenate_indexed_table.q.out Tue Apr 7 21:51:52 2015
@@ -76,7 +76,7 @@ PREHOOK: query: show indexes on src_rc_c
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: show indexes on src_rc_concatenate_test
POSTHOOK: type: SHOWINDEXES
-src_rc_concatenate_test_index src_rc_concatenate_test key default.default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact
+src_rc_concatenate_test_index src_rc_concatenate_test key default__src_rc_concatenate_test_src_rc_concatenate_test_index__ compact
PREHOOK: query: alter table src_rc_concatenate_test concatenate
PREHOOK: type: ALTER_TABLE_MERGE
PREHOOK: Input: default@src_rc_concatenate_test
@@ -215,7 +215,7 @@ PREHOOK: query: show indexes on src_rc_c
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: show indexes on src_rc_concatenate_test_part
POSTHOOK: type: SHOWINDEXES
-src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default.default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact
+src_rc_concatenate_test_part_index src_rc_concatenate_test_part key default__src_rc_concatenate_test_part_src_rc_concatenate_test_part_index__ compact
PREHOOK: query: alter table src_rc_concatenate_test_part partition (ds='2011') concatenate
PREHOOK: type: ALTER_PARTITION_MERGE
PREHOOK: Input: default@src_rc_concatenate_test_part
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_select.q.out Tue Apr 7 21:51:52 2015
@@ -395,7 +395,7 @@ STAGE PLANS:
alias: alltypes_orc
Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: CAST( 'hello' AS CHAR(5) (type: char(5))
+ expressions: 'hello' (type: char(5))
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
@@ -416,7 +416,7 @@ STAGE PLANS:
alias: alltypes_orc
Statistics: Num rows: 2 Data size: 1686 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: CAST( 'hello' AS varchar(5)) (type: varchar(5))
+ expressions: 'hello' (type: varchar(5))
outputColumnNames: _col0
Statistics: Num rows: 2 Data size: 178 Basic stats: COMPLETE Column stats: COMPLETE
ListSink
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/groupby3_map.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/groupby3_map.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/groupby3_map.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/groupby3_map.q.out Tue Apr 7 21:51:52 2015
@@ -128,12 +128,32 @@ POSTHOOK: Lineage: dest1.c6 EXPRESSION [
POSTHOOK: Lineage: dest1.c7 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c8 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
POSTHOOK: Lineage: dest1.c9 EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
-PREHOOK: query: SELECT dest1.* FROM dest1
+PREHOOK: query: SELECT
+c1,
+c2,
+round(c3, 11) c3,
+c4,
+c5,
+round(c6, 11) c6,
+round(c7, 11) c7,
+round(c8, 5) c8,
+round(c9, 9) c9
+FROM dest1
PREHOOK: type: QUERY
PREHOOK: Input: default@dest1
#### A masked pattern was here ####
-POSTHOOK: query: SELECT dest1.* FROM dest1
+POSTHOOK: query: SELECT
+c1,
+c2,
+round(c3, 11) c3,
+c4,
+c5,
+round(c6, 11) c6,
+round(c7, 11) c7,
+round(c8, 5) c8,
+round(c9, 9) c9
+FROM dest1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@dest1
#### A masked pattern was here ####
-130091.0 260.182 256.10355987055016 98.0 0.0 142.9268095075238 143.06995106518906 20428.072876 20469.01089779559
+130091.0 260.182 256.10355987055 98.0 0.0 142.92680950752 143.06995106519 20428.07288 20469.010897796
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auth.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auth.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auth.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auth.q.out Tue Apr 7 21:51:52 2015
@@ -24,7 +24,7 @@ PREHOOK: query: SHOW INDEXES ON foobar
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: SHOW INDEXES ON foobar
POSTHOOK: type: SHOWINDEXES
-srcpart_auth_index foobar key default.default__foobar_srcpart_auth_index__ bitmap
+srcpart_auth_index foobar key default__foobar_srcpart_auth_index__ bitmap
PREHOOK: query: grant select on table foobar to user hive_test_user
PREHOOK: type: GRANT_PRIVILEGE
PREHOOK: Output: default@foobar
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto.q.out Tue Apr 7 21:51:52 2015
@@ -146,7 +146,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean)
Filter Operator
predicate: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_file_format.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_file_format.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_file_format.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_file_format.q.out Tue Apr 7 21:51:52 2015
@@ -40,7 +40,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: (UDFToDouble(key) = 86.0) (type: boolean)
Filter Operator
predicate: (UDFToDouble(key) = 86.0) (type: boolean)
@@ -156,7 +156,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: (UDFToDouble(key) = 86.0) (type: boolean)
Filter Operator
predicate: (UDFToDouble(key) = 86.0) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables.q.out Tue Apr 7 21:51:52 2015
@@ -211,7 +211,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__srcpart_srcpart_index__
+ alias: default__srcpart_srcpart_index__
filterExpr: (((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Filter Operator
predicate: (((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
@@ -308,7 +308,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: (((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Filter Operator
predicate: (((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_mult_tables_compact.q.out Tue Apr 7 21:51:52 2015
@@ -221,7 +221,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__srcpart_srcpart_index__
+ alias: default__srcpart_srcpart_index__
filterExpr: ((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) (type: boolean)
Filter Operator
predicate: ((((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (UDFToDouble(key) > 80.0)) and (UDFToDouble(key) < 100.0)) (type: boolean)
@@ -339,7 +339,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: ((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) (type: boolean)
Filter Operator
predicate: ((((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (UDFToDouble(key) > 70.0)) and (UDFToDouble(key) < 90.0)) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_multiple.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_multiple.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_multiple.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_multiple.q.out Tue Apr 7 21:51:52 2015
@@ -60,7 +60,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_key_index__
+ alias: default__src_src_key_index__
filterExpr: (UDFToDouble(key) = 86.0) (type: boolean)
Filter Operator
predicate: (UDFToDouble(key) = 86.0) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_partitioned.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_partitioned.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_partitioned.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_partitioned.q.out Tue Apr 7 21:51:52 2015
@@ -54,7 +54,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__srcpart_src_part_index__
+ alias: default__srcpart_src_part_index__
filterExpr: ((UDFToDouble(key) = 86.0) and (ds = '2008-04-09')) (type: boolean)
Filter Operator
predicate: (UDFToDouble(key) = 86.0) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_self_join.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_self_join.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_self_join.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_self_join.q.out Tue Apr 7 21:51:52 2015
@@ -131,7 +131,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Filter Operator
predicate: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
@@ -229,7 +229,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Filter Operator
predicate: (((UDFToDouble(key) > 70.0) and (UDFToDouble(key) < 90.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_update.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_update.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_update.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_update.q.out Tue Apr 7 21:51:52 2015
@@ -219,7 +219,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__temp_temp_index__
+ alias: default__temp_temp_index__
filterExpr: (UDFToDouble(key) = 86.0) (type: boolean)
Filter Operator
predicate: (UDFToDouble(key) = 86.0) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_auto_partitioned.q.out Tue Apr 7 21:51:52 2015
@@ -52,7 +52,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__srcpart_src_part_index__
+ alias: default__srcpart_src_part_index__
filterExpr: ((UDFToDouble(key) = 86.0) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Filter Operator
predicate: ((UDFToDouble(key) = 86.0) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_compression.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_compression.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_compression.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_bitmap_compression.q.out Tue Apr 7 21:51:52 2015
@@ -38,7 +38,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Filter Operator
predicate: (((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) and (not EWAH_BITMAP_EMPTY(_bitmaps))) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_compression.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_compression.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_compression.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_compression.q.out Tue Apr 7 21:51:52 2015
@@ -42,7 +42,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__src_src_index__
+ alias: default__src_src_index__
filterExpr: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean)
Filter Operator
predicate: ((UDFToDouble(key) > 80.0) and (UDFToDouble(key) < 100.0)) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_serde.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_serde.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_serde.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_serde.q.out Tue Apr 7 21:51:52 2015
@@ -134,7 +134,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__doctors_doctors_index__
+ alias: default__doctors_doctors_index__
filterExpr: (number > 6) (type: boolean)
Filter Operator
predicate: (number > 6) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/index_skewtable.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/index_skewtable.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/index_skewtable.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/index_skewtable.q.out Tue Apr 7 21:51:52 2015
@@ -92,7 +92,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__kv_kv_index__
+ alias: default__kv_kv_index__
filterExpr: (value > '15') (type: boolean)
Filter Operator
predicate: (value > '15') (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx.q.out Tue Apr 7 21:51:52 2015
@@ -253,7 +253,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
@@ -572,7 +572,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
@@ -746,7 +746,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: lastyear:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: lastyear:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (year(l_shipdate) = 1997) (type: boolean)
@@ -830,7 +830,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: thisyear:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: thisyear:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (year(l_shipdate) = 1998) (type: boolean)
@@ -900,7 +900,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: null-subquery1:dummy-subquery1:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: null-subquery1:dummy-subquery1:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
@@ -1063,7 +1063,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__tbl_tbl_key_idx__
+ alias: default__tbl_tbl_key_idx__
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
@@ -1161,7 +1161,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__tbl_tbl_key_idx__
+ alias: default__tbl_tbl_key_idx__
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
@@ -2252,7 +2252,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__tblpart_tbl_part_index__
+ alias: default__tblpart_tbl_part_index__
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < 10) (type: boolean)
@@ -2445,7 +2445,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: default.default__tbl_tbl_key_idx__
+ alias: default__tbl_tbl_key_idx__
Statistics: Num rows: 6 Data size: 430 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_1.q.out Tue Apr 7 21:51:52 2015
@@ -253,7 +253,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
@@ -572,7 +572,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: year(l_shipdate) (type: int), month(l_shipdate) (type: int), l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
@@ -746,7 +746,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: lastyear:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: lastyear:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (year(l_shipdate) = 1997) (type: boolean)
@@ -830,7 +830,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: thisyear:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: thisyear:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (year(l_shipdate) = 1998) (type: boolean)
@@ -900,7 +900,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: null-subquery1:$hdt$_0-subquery1:$hdt$_0:default.default__lineitem_ix_lineitem_ix_lshipdate_idx__
+ alias: null-subquery1:$hdt$_0-subquery1:$hdt$_0:default__lineitem_ix_lineitem_ix_lshipdate_idx__
Statistics: Num rows: 95 Data size: 8960 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_shipdate (type: string), _count_of_l_shipdate (type: bigint)
@@ -1063,7 +1063,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__tbl_tbl_key_idx__
+ alias: $hdt$_0:default__tbl_tbl_key_idx__
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
@@ -1161,7 +1161,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__tbl_tbl_key_idx__
+ alias: $hdt$_0:default__tbl_tbl_key_idx__
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
@@ -2260,7 +2260,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__tblpart_tbl_part_index__
+ alias: $hdt$_0:default__tblpart_tbl_part_index__
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < 10) (type: boolean)
@@ -2453,7 +2453,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__tbl_tbl_key_idx__
+ alias: $hdt$_0:default__tbl_tbl_key_idx__
Statistics: Num rows: 6 Data size: 430 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: int), _count_of_key (type: bigint)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/ql_rewrite_gbtoidx_cbo_2.q.out Tue Apr 7 21:51:52 2015
@@ -176,7 +176,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_orderkey (type: int), _count_of_l_orderkey (type: bigint)
@@ -1223,7 +1223,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_orderkey (type: int), _count_of_l_orderkey (type: bigint)
@@ -1907,7 +1907,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:$hdt$_0:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: $hdt$_0:$hdt$_0:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (l_orderkey < 7) (type: boolean)
@@ -2072,7 +2072,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: taba:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: taba:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (l_orderkey < 7) (type: boolean)
@@ -2148,7 +2148,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: tabb:default.default__src_src_key_idx__
+ alias: tabb:default__src_src_key_idx__
Statistics: Num rows: 309 Data size: 23663 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string), _count_of_key (type: bigint)
@@ -2814,7 +2814,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: taba:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: taba:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (l_orderkey < 7) (type: boolean)
@@ -2890,7 +2890,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: tabb:default.default__src_src_key_idx__
+ alias: tabb:default__src_src_key_idx__
Statistics: Num rows: 309 Data size: 23663 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (key < '2') (type: boolean)
@@ -3192,7 +3192,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: l_orderkey (type: int), (l_orderkey + 1) (type: int), _count_of_l_orderkey (type: bigint)
@@ -3282,7 +3282,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: $hdt$_0:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: (l_orderkey + 2) (type: int), l_orderkey (type: int), _count_of_l_orderkey (type: bigint)
@@ -3404,7 +3404,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0:default.default__lineitem_ix_lineitem_ix_l_orderkey_idx__
+ alias: $hdt$_0-subquery1:$hdt$_0-subquery1:$hdt$_0:default__lineitem_ix_lineitem_ix_l_orderkey_idx__
Statistics: Num rows: 26 Data size: 2604 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (l_orderkey < 7) (type: boolean)
@@ -3495,7 +3495,7 @@ STAGE PLANS:
Map Reduce
Map Operator Tree:
TableScan
- alias: $hdt$_0-subquery2:$hdt$_0-subquery2:$hdt$_0:default.default__lineitem_ix_lineitem_ix_l_partkey_idx__
+ alias: $hdt$_0-subquery2:$hdt$_0-subquery2:$hdt$_0:default__lineitem_ix_lineitem_ix_l_partkey_idx__
Statistics: Num rows: 100 Data size: 8937 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (l_partkey < 10) (type: boolean)
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_edge_cases.q.out Tue Apr 7 21:51:52 2015
@@ -118,10 +118,10 @@ PREHOOK: query: SHOW INDEXES ON show_idx
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: SHOW INDEXES ON show_idx_full
POSTHOOK: type: SHOWINDEXES
-idx_1 show_idx_full key default.default__show_idx_full_idx_1__ compact
-idx_2 show_idx_full value1 default.default__show_idx_full_idx_2__ compact
-idx_comment show_idx_full value2 default.default__show_idx_full_idx_comment__ compact index comment
-idx_compound show_idx_full key, value1 default.default__show_idx_full_idx_compound__ compact
+idx_1 show_idx_full key default__show_idx_full_idx_1__ compact
+idx_2 show_idx_full value1 default__show_idx_full_idx_2__ compact
+idx_comment show_idx_full value2 default__show_idx_full_idx_comment__ compact index comment
+idx_compound show_idx_full key, value1 default__show_idx_full_idx_compound__ compact
PREHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_empty
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: EXPLAIN SHOW INDEXES ON show_idx_empty
Modified: hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_syntax.q.out
URL: http://svn.apache.org/viewvc/hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_syntax.q.out?rev=1671959&r1=1671958&r2=1671959&view=diff
==============================================================================
--- hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_syntax.q.out (original)
+++ hive/branches/cbo/ql/src/test/results/clientpositive/show_indexes_syntax.q.out Tue Apr 7 21:51:52 2015
@@ -53,7 +53,7 @@ PREHOOK: query: SHOW INDEX ON show_idx_t
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: SHOW INDEX ON show_idx_t1
POSTHOOK: type: SHOWINDEXES
-idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact
+idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact
PREHOOK: query: EXPLAIN
SHOW INDEXES ON show_idx_t1
PREHOOK: type: SHOWINDEXES
@@ -79,7 +79,7 @@ PREHOOK: query: SHOW INDEXES ON show_idx
PREHOOK: type: SHOWINDEXES
POSTHOOK: query: SHOW INDEXES ON show_idx_t1
POSTHOOK: type: SHOWINDEXES
-idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact
+idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact
PREHOOK: query: EXPLAIN
SHOW FORMATTED INDEXES ON show_idx_t1
PREHOOK: type: SHOWINDEXES
@@ -108,7 +108,7 @@ POSTHOOK: type: SHOWINDEXES
idx_name tab_name col_names idx_tab_name idx_type comment
-idx_t1 show_idx_t1 key default.default__show_idx_t1_idx_t1__ compact
+idx_t1 show_idx_t1 key default__show_idx_t1_idx_t1__ compact
PREHOOK: query: DROP TABLE show_idx_t1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@show_idx_t1