You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/03/22 06:01:45 UTC
svn commit: r1580168 [1/2] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/
java/org/apache/hadoop/hive/ql/parse/ java/org/apache/hadoop/hive/ql/plan/
java/org/apache/hadoop/hive/ql/ppd/ test/queri...
Author: hashutosh
Date: Sat Mar 22 05:01:42 2014
New Revision: 1580168
URL: http://svn.apache.org/r1580168
Log:
HIVE-4293 : Predicates following UDTF operator are removed by PPD (Navis via Harish Butani)
Added:
hive/trunk/ql/src/test/queries/clientpositive/ppd_join4.q
hive/trunk/ql/src/test/queries/clientpositive/ppd_udtf.q
hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_udtf.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java
hive/trunk/ql/src/test/queries/clientpositive/lateral_view_ppd.q
hive/trunk/ql/src/test/queries/clientpositive/ppd_transform.q
hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_gby.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out
hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out
hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out
hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java Sat Mar 22 05:01:42 2014
@@ -78,8 +78,8 @@ public class LateralViewJoinOperator ext
// The expected tags from the parent operators. See processOp() before
// changing the tags.
- public static final int SELECT_TAG = 0;
- public static final int UDTF_TAG = 1;
+ public static final byte SELECT_TAG = 0;
+ public static final byte UDTF_TAG = 1;
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Sat Mar 22 05:01:42 2014
@@ -55,6 +55,10 @@ import org.apache.hadoop.mapred.OutputCo
public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc>
implements Serializable, TopNHash.BinaryCollector {
+ static {
+ PTFUtils.makeTransient(ReduceSinkOperator.class, "inputAliases");
+ }
+
private static final long serialVersionUID = 1L;
protected transient OutputCollector out;
@@ -84,14 +88,14 @@ public class ReduceSinkOperator extends
protected transient byte[] tagByte = new byte[1];
transient protected int numDistributionKeys;
transient protected int numDistinctExprs;
- transient String inputAlias; // input alias of this RS for join (used for PPD)
+ transient String[] inputAliases; // input aliases of this RS for join (used for PPD)
- public void setInputAlias(String inputAlias) {
- this.inputAlias = inputAlias;
+ public void setInputAliases(String[] inputAliases) {
+ this.inputAliases = inputAliases;
}
- public String getInputAlias() {
- return inputAlias;
+ public String[] getInputAliases() {
+ return inputAliases;
}
public void setOutputCollector(OutputCollector _out) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Sat Mar 22 05:01:42 2014
@@ -474,39 +474,38 @@ public final class ColumnPrunerProcFacto
List<String> cols = cppCtx.genColLists(op);
Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap();
-
// As columns go down the DAG, the LVJ will transform internal column
// names from something like 'key' to '_col0'. Because of this, we need
// to undo this transformation using the column expression map as the
// column names propagate up the DAG.
- List<String> colsAfterReplacement = new ArrayList<String>();
- for (String col : cols) {
- if (colExprMap.containsKey(col)) {
- ExprNodeDesc expr = colExprMap.get(col);
- colsAfterReplacement.addAll(expr.getCols());
- } else {
- colsAfterReplacement.add(col);
- }
- }
+
// this is SEL(*) cols + UDTF cols
List<String> outputCols = op.getConf().getOutputInternalColNames();
- if (outputCols.size() != cols.size()) {
- // cause we cannot prune columns from UDTF branch currently, extract
- // columns from SEL(*) branch only and append all columns from UDTF branch to it
- ArrayList<String> newColNames = new ArrayList<String>();
- for (String col : cols) {
- int index = outputCols.indexOf(col);
- // colExprMap.size() == size of cols from SEL(*) branch
- if (index >= 0 && index < colExprMap.size()) {
- newColNames.add(col);
- }
- }
- newColNames.addAll(outputCols.subList(colExprMap.size(), outputCols.size()));
- op.getConf().setOutputInternalColNames(newColNames);
- }
- cppCtx.getPrunedColLists().put(op,
- colsAfterReplacement);
+ // cause we cannot prune columns from UDTF branch currently, extract
+ // columns from SEL(*) branch only and append all columns from UDTF branch to it
+ int numSelColumns = op.getConf().getNumSelColumns();
+
+ List<String> colsAfterReplacement = new ArrayList<String>();
+ ArrayList<String> newColNames = new ArrayList<String>();
+ for (String col : cols) {
+ int index = outputCols.indexOf(col);
+ // colExprMap.size() == size of cols from SEL(*) branch
+ if (index >= 0 && index < numSelColumns) {
+ ExprNodeDesc transformed = colExprMap.get(col);
+ Utilities.mergeUniqElems(colsAfterReplacement, transformed.getCols());
+ newColNames.add(col);
+ }
+ }
+ // update number of columns from sel(*)
+ op.getConf().setNumSelColumns(newColNames.size());
+
+ // add all UDTF columns
+ // following SEL will do CP for columns from UDTF, not adding SEL in here
+ newColNames.addAll(outputCols.subList(numSelColumns, outputCols.size()));
+ op.getConf().setOutputInternalColNames(newColNames);
+
+ cppCtx.getPrunedColLists().put(op, colsAfterReplacement);
return null;
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Sat Mar 22 05:01:42 2014
@@ -6587,7 +6587,7 @@ public class SemanticAnalyzer extends Ba
@SuppressWarnings("nls")
private Operator genJoinReduceSinkChild(QB qb, QBJoinTree joinTree,
- Operator child, String srcName, int pos) throws SemanticException {
+ Operator child, String[] srcs, int pos) throws SemanticException {
RowResolver inputRS = opParseCtx.get(child).getRowResolver();
RowResolver outputRS = new RowResolver();
ArrayList<String> outputColumns = new ArrayList<String>();
@@ -6647,7 +6647,7 @@ public class SemanticAnalyzer extends Ba
reduceKeys.size(), numReds), new RowSchema(outputRS
.getColumnInfos()), child), outputRS);
rsOp.setColumnExprMap(colExprMap);
- rsOp.setInputAlias(srcName);
+ rsOp.setInputAliases(srcs);
return rsOp;
}
@@ -6666,7 +6666,8 @@ public class SemanticAnalyzer extends Ba
for (ASTNode cond : filter) {
joinSrcOp = genFilterPlan(qb, cond, joinSrcOp);
}
- joinSrcOp = genJoinReduceSinkChild(qb, joinTree, joinSrcOp, null, 0);
+ String[] leftAliases = joinTree.getLeftAliases();
+ joinSrcOp = genJoinReduceSinkChild(qb, joinTree, joinSrcOp, leftAliases, 0);
}
Operator[] srcOps = new Operator[joinTree.getBaseSrc().length];
@@ -6698,7 +6699,7 @@ public class SemanticAnalyzer extends Ba
}
// generate a ReduceSink operator for the join
- srcOps[pos] = genJoinReduceSinkChild(qb, joinTree, srcOp, src, pos);
+ srcOps[pos] = genJoinReduceSinkChild(qb, joinTree, srcOp, new String[]{src}, pos);
pos++;
} else {
assert pos == 0;
@@ -9073,6 +9074,7 @@ public class SemanticAnalyzer extends Ba
Operator allPath = putOpInsertMap(OperatorFactory.getAndMakeChild(
new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()),
lvForward), allPathRR);
+ int allColumns = allPathRR.getColumnInfos().size();
// Get the UDTF Path
QB blankQb = new QB(null, null, false);
Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
@@ -9092,8 +9094,6 @@ public class SemanticAnalyzer extends Ba
RowResolver lateralViewRR = new RowResolver();
ArrayList<String> outputInternalColNames = new ArrayList<String>();
- LVmergeRowResolvers(allPathRR, lateralViewRR, outputInternalColNames);
- LVmergeRowResolvers(udtfPathRR, lateralViewRR, outputInternalColNames);
// For PPD, we need a column to expression map so that during the walk,
// the processor knows how to transform the internal col names.
@@ -9101,17 +9101,11 @@ public class SemanticAnalyzer extends Ba
// LVmerge.. in the above order
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
- int i = 0;
- for (ColumnInfo c : allPathRR.getColumnInfos()) {
- String internalName = getColumnInternalName(i);
- i++;
- colExprMap.put(internalName,
- new ExprNodeColumnDesc(c.getType(), c.getInternalName(),
- c.getTabAlias(), c.getIsVirtualCol()));
- }
+ LVmergeRowResolvers(allPathRR, lateralViewRR, colExprMap, outputInternalColNames);
+ LVmergeRowResolvers(udtfPathRR, lateralViewRR, colExprMap, outputInternalColNames);
Operator lateralViewJoin = putOpInsertMap(OperatorFactory
- .getAndMakeChild(new LateralViewJoinDesc(outputInternalColNames),
+ .getAndMakeChild(new LateralViewJoinDesc(allColumns, outputInternalColNames),
new RowSchema(lateralViewRR.getColumnInfos()), allPath,
udtfPath), lateralViewRR);
lateralViewJoin.setColumnExprMap(colExprMap);
@@ -9134,7 +9128,7 @@ public class SemanticAnalyzer extends Ba
* the same order as in the dest row resolver
*/
private void LVmergeRowResolvers(RowResolver source, RowResolver dest,
- ArrayList<String> outputInternalColNames) {
+ Map<String, ExprNodeDesc> colExprMap, ArrayList<String> outputInternalColNames) {
for (ColumnInfo c : source.getColumnInfos()) {
String internalName = getColumnInternalName(outputInternalColNames.size());
outputInternalColNames.add(internalName);
@@ -9144,6 +9138,8 @@ public class SemanticAnalyzer extends Ba
String tableAlias = tableCol[0];
String colAlias = tableCol[1];
dest.put(tableAlias, colAlias, newCol);
+ colExprMap.put(internalName, new ExprNodeColumnDesc(c.getType(), c.getInternalName(),
+ c.getTabAlias(), c.getIsVirtualCol()));
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/LateralViewJoinDesc.java Sat Mar 22 05:01:42 2014
@@ -29,12 +29,14 @@ import java.util.ArrayList;
public class LateralViewJoinDesc extends AbstractOperatorDesc {
private static final long serialVersionUID = 1L;
+ private int numSelColumns;
private ArrayList<String> outputInternalColNames;
public LateralViewJoinDesc() {
}
- public LateralViewJoinDesc(ArrayList<String> outputInternalColNames) {
+ public LateralViewJoinDesc(int numSelColumns, ArrayList<String> outputInternalColNames) {
+ this.numSelColumns = numSelColumns;
this.outputInternalColNames = outputInternalColNames;
}
@@ -46,4 +48,12 @@ public class LateralViewJoinDesc extends
public ArrayList<String> getOutputInternalColNames() {
return outputInternalColNames;
}
+
+ public int getNumSelColumns() {
+ return numSelColumns;
+ }
+
+ public void setNumSelColumns(int numSelColumns) {
+ this.numSelColumns = numSelColumns;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/ExprWalkerInfo.java Sat Mar 22 05:01:42 2014
@@ -19,9 +19,11 @@ package org.apache.hadoop.hive.ql.ppd;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -289,6 +291,21 @@ public class ExprWalkerInfo implements N
return nonFinalPreds;
}
+ public Map<String, List<ExprNodeDesc>> getResidualPredicates(boolean clear) {
+ Map<String, List<ExprNodeDesc>> oldExprs = new HashMap<String, List<ExprNodeDesc>>();
+ for (Map.Entry<String, List<ExprNodeDesc>> entry : nonFinalPreds.entrySet()) {
+ List<ExprNodeDesc> converted = new ArrayList<ExprNodeDesc>();
+ for (ExprNodeDesc newExpr : entry.getValue()) {
+ converted.add(newToOldExprMap.get(newExpr));
+ }
+ oldExprs.put(entry.getKey(), converted);
+ }
+ if (clear) {
+ nonFinalPreds.clear();
+ }
+ return oldExprs;
+ }
+
/**
* Merges the specified pushdown predicates with the current class.
*
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/OpProcFactory.java Sat Mar 22 05:01:42 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
@@ -81,6 +82,32 @@ public final class OpProcFactory {
protected static final Log LOG = LogFactory.getLog(OpProcFactory.class
.getName());
+ private static ExprWalkerInfo getChildWalkerInfo(Operator<?> current, OpWalkerInfo owi) {
+ if (current.getNumChild() == 0) {
+ return null;
+ }
+ if (current.getNumChild() > 1) {
+ // ppd for multi-insert query is not yet implemented
+ // no-op for leafs
+ for (Operator<?> child : current.getChildOperators()) {
+ removeCandidates(child, owi); // remove candidated filters on this branch
+ }
+ return null;
+ }
+ return owi.getPrunedPreds(current.getChildOperators().get(0));
+ }
+
+ private static void removeCandidates(Operator<?> operator, OpWalkerInfo owi) {
+ if (operator instanceof FilterOperator) {
+ owi.getCandidateFilterOps().remove(operator);
+ }
+ if (operator.getChildOperators() != null) {
+ for (Operator<?> child : operator.getChildOperators()) {
+ removeCandidates(child, owi);
+ }
+ }
+ }
+
/**
* Processor for Script Operator Prevents any predicates being pushed.
*/
@@ -96,7 +123,8 @@ public final class OpProcFactory {
// same with LIMIT op
// create a filter with all children predicates
OpWalkerInfo owi = (OpWalkerInfo) procCtx;
- if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+ ExprWalkerInfo childInfo = getChildWalkerInfo((Operator<?>) nd, owi);
+ if (childInfo != null && HiveConf.getBoolVar(owi.getParseContext().getConf(),
HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
ExprWalkerInfo unpushedPreds = mergeChildrenPred(nd, owi, null, false);
return createFilter((Operator)nd, unpushedPreds, owi);
@@ -110,10 +138,17 @@ public final class OpProcFactory {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- LOG.info("Processing for " + nd.getName() + "("
- + ((Operator) nd).getIdentifier() + ")");
- //Predicates for UDTF wont be candidates for its children. So, nothing to
- //optimize here. See lateral_view_ppd.q for example.
+ super.process(nd, stack, procCtx, nodeOutputs);
+ OpWalkerInfo owi = (OpWalkerInfo) procCtx;
+ ExprWalkerInfo prunedPred = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd);
+ if (prunedPred == null) {
+ return null;
+ }
+ Map<String, List<ExprNodeDesc>> candidates = prunedPred.getFinalCandidates();
+ if (candidates != null && !candidates.isEmpty()) {
+ createFilter((Operator)nd, prunedPred, owi);
+ candidates.clear();
+ }
return null;
}
@@ -155,7 +190,7 @@ public final class OpProcFactory {
+ ((Operator) nd).getIdentifier() + ")");
OpWalkerInfo owi = (OpWalkerInfo) procCtx;
TableScanOperator tsOp = (TableScanOperator) nd;
- mergeWithChildrenPred(tsOp, owi, null, null, false);
+ mergeWithChildrenPred(tsOp, owi, null, null);
ExprWalkerInfo pushDownPreds = owi.getPrunedPreds(tsOp);
return createFilter(tsOp, pushDownPreds, owi);
}
@@ -204,7 +239,7 @@ public final class OpProcFactory {
owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, ewi);
}
// merge it with children predicates
- boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, ewi, null, false);
+ boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, ewi, null);
if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
if (hasUnpushedPredicates) {
@@ -220,20 +255,18 @@ public final class OpProcFactory {
* Determines predicates for which alias can be pushed to it's parents. See
* the comments for getQualifiedAliases function.
*/
- public static class JoinPPD extends DefaultPPD implements NodeProcessor {
+ public static class JoinerPPD extends DefaultPPD implements NodeProcessor {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
LOG.info("Processing for " + nd.getName() + "("
+ ((Operator) nd).getIdentifier() + ")");
OpWalkerInfo owi = (OpWalkerInfo) procCtx;
- Set<String> aliases = getQualifiedAliases((JoinOperator) nd, owi
- .getRowResolver(nd));
+ Set<String> aliases = getAliases(nd, owi);
// we pass null for aliases here because mergeWithChildrenPred filters
// aliases in the children node context and we need to filter them in
// the current JoinOperator's context
- boolean hasUnpushedPredicates =
- mergeWithChildrenPred(nd, owi, null, null, false);
+ mergeWithChildrenPred(nd, owi, null, null);
ExprWalkerInfo prunePreds =
owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd);
if (prunePreds != null) {
@@ -255,31 +288,42 @@ public final class OpProcFactory {
}
prunePreds.getFinalCandidates().remove(alias);
}
- if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
- HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) {
- applyFilterTransitivity((JoinOperator) nd, owi);
- }
- if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
- HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
- // Here, we add all the "non-final candidiates", ie. the predicates
- // rejected from pushdown through this operator to unpushedPreds
- // and pass it to createFilter
- ExprWalkerInfo unpushedPreds = new ExprWalkerInfo();
- for (Entry<String, List<ExprNodeDesc>> entry :
- prunePreds.getNonFinalCandidates().entrySet()) {
- for (ExprNodeDesc expr : entry.getValue()) {
- assert prunePreds.getNewToOldExprMap().containsKey(expr);
- ExprNodeDesc oldExpr = prunePreds.getNewToOldExprMap().get(expr);
- unpushedPreds.addAlias(oldExpr, entry.getKey());
- unpushedPreds.addFinalCandidate(oldExpr);
- }
- }
- return createFilter((Operator)nd, unpushedPreds, owi);
- }
+ return handlePredicates(nd, prunePreds, owi);
}
return null;
}
+ protected Set<String> getAliases(Node nd, OpWalkerInfo owi) throws SemanticException {
+ return owi.getRowResolver(nd).getTableNames();
+ }
+
+ protected Object handlePredicates(Node nd, ExprWalkerInfo prunePreds, OpWalkerInfo owi)
+ throws SemanticException {
+ if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+ HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
+ return createFilter((Operator)nd, prunePreds.getResidualPredicates(true), owi);
+ }
+ return null;
+ }
+ }
+
+ public static class JoinPPD extends JoinerPPD {
+
+ @Override
+ protected Set<String> getAliases(Node nd, OpWalkerInfo owi) {
+ return getQualifiedAliases((JoinOperator) nd, owi.getRowResolver(nd));
+ }
+
+ @Override
+ protected Object handlePredicates(Node nd, ExprWalkerInfo prunePreds, OpWalkerInfo owi)
+ throws SemanticException {
+ if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+ HiveConf.ConfVars.HIVEPPDRECOGNIZETRANSITIVITY)) {
+ applyFilterTransitivity((JoinOperator) nd, owi);
+ }
+ return super.handlePredicates(nd, prunePreds, owi);
+ }
+
/**
* Adds additional pushdown predicates for a join operator by replicating
* filters transitively over all the equijoin conditions.
@@ -495,66 +539,56 @@ public final class OpProcFactory {
}
/**
- * Processor for ReduceSink operator.
- *
+ * Default processor which just merges its children.
*/
- public static class ReduceSinkPPD extends DefaultPPD implements NodeProcessor {
+ public static class DefaultPPD implements NodeProcessor {
+
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
LOG.info("Processing for " + nd.getName() + "("
+ ((Operator) nd).getIdentifier() + ")");
- ReduceSinkOperator rs = (ReduceSinkOperator) nd;
OpWalkerInfo owi = (OpWalkerInfo) procCtx;
- Set<String> aliases;
- boolean ignoreAliases = false;
- if (rs.getInputAlias() != null) {
- aliases = new HashSet<String>(Arrays.asList(rs.getInputAlias()));
- } else {
- aliases = owi.getRowResolver(nd).getTableNames();
- if (aliases.size() == 1 && aliases.contains("")) {
- // Reduce sink of group by operator
- ignoreAliases = true;
- }
- }
- boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, aliases, ignoreAliases);
- if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
+ Set<String> includes = getQualifiedAliases((Operator<?>) nd, owi);
+ boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, includes);
+ if (hasUnpushedPredicates && HiveConf.getBoolVar(owi.getParseContext().getConf(),
HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
- if (hasUnpushedPredicates) {
- Operator<? extends OperatorDesc> op =
- (Operator<? extends OperatorDesc>) nd;
- Operator<? extends OperatorDesc> childOperator = op.getChildOperators().get(0);
- if(childOperator.getParentOperators().size()==1) {
- owi.getCandidateFilterOps().clear();
+ if (includes != null || nd instanceof ReduceSinkOperator) {
+ owi.getCandidateFilterOps().clear();
+ } else {
+ ExprWalkerInfo pruned = owi.getPrunedPreds((Operator<? extends OperatorDesc>) nd);
+ Map<String, List<ExprNodeDesc>> residual = pruned.getResidualPredicates(true);
+ if (residual != null && !residual.isEmpty()) {
+ createFilter((Operator) nd, residual, owi);
+ pruned.getNonFinalCandidates().clear();
}
}
}
return null;
}
- }
-
- /**
- * Default processor which just merges its children.
- */
- public static class DefaultPPD implements NodeProcessor {
-
- @Override
- public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
- Object... nodeOutputs) throws SemanticException {
- LOG.info("Processing for " + nd.getName() + "("
- + ((Operator) nd).getIdentifier() + ")");
- OpWalkerInfo owi = (OpWalkerInfo) procCtx;
- boolean hasUnpushedPredicates = mergeWithChildrenPred(nd, owi, null, null, false);
- if (HiveConf.getBoolVar(owi.getParseContext().getConf(),
- HiveConf.ConfVars.HIVEPPDREMOVEDUPLICATEFILTERS)) {
- if (hasUnpushedPredicates) {
- ExprWalkerInfo unpushedPreds = mergeChildrenPred(nd, owi, null, false);
- return createFilter((Operator)nd, unpushedPreds, owi);
- }
+ // RS for join, SEL(*) for lateral view
+ // SEL for union does not count (should be copied to both sides)
+ private Set<String> getQualifiedAliases(Operator<?> operator, OpWalkerInfo owi) {
+ if (operator.getNumChild() != 1) {
+ return null;
}
- return null;
+ Operator<?> child = operator.getChildOperators().get(0);
+ if (!(child instanceof JoinOperator || child instanceof LateralViewJoinOperator)) {
+ return null;
+ }
+ if (operator instanceof ReduceSinkOperator &&
+ ((ReduceSinkOperator)operator).getInputAliases() != null) {
+ String[] aliases = ((ReduceSinkOperator)operator).getInputAliases();
+ return new HashSet<String>(Arrays.asList(aliases));
+ }
+ Set<String> includes = owi.getRowResolver(operator).getTableNames();
+ if (includes.size() == 1 && includes.contains("")) {
+ // Reduce sink of group by operator
+ return null;
+ }
+ return includes;
}
/**
@@ -585,39 +619,22 @@ public final class OpProcFactory {
* @param aliases
* aliases that this operator can pushdown. null means that all
* aliases can be pushed down
- * @param ignoreAliases
* @throws SemanticException
*/
protected boolean mergeWithChildrenPred(Node nd, OpWalkerInfo owi,
- ExprWalkerInfo ewi, Set<String> aliases, boolean ignoreAliases)
- throws SemanticException {
- boolean hasUnpushedPredicates = false;
- Operator<?> current = (Operator<?>) nd;
- List<Operator<?>> children = current.getChildOperators();
- if (children == null || children.isEmpty()) {
- return hasUnpushedPredicates;
- }
- if (children.size() > 1) {
- // ppd for multi-insert query is not yet implemented
- // no-op for leafs
- for (Operator<?> child : children) {
- removeCandidates(child, owi); // remove candidated filters on this branch
- }
- return hasUnpushedPredicates;
- }
- Operator<? extends OperatorDesc> op =
- (Operator<? extends OperatorDesc>) nd;
- ExprWalkerInfo childPreds = owi.getPrunedPreds(children.get(0));
+ ExprWalkerInfo ewi, Set<String> aliases) throws SemanticException {
+ Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
+ ExprWalkerInfo childPreds = getChildWalkerInfo(op, owi);
if (childPreds == null) {
- return hasUnpushedPredicates;
+ return false;
}
if (ewi == null) {
ewi = new ExprWalkerInfo();
}
+ boolean hasUnpushedPredicates = false;
for (Entry<String, List<ExprNodeDesc>> e : childPreds
.getFinalCandidates().entrySet()) {
- if (ignoreAliases || aliases == null || aliases.contains(e.getKey())
- || e.getKey() == null) {
+ if (aliases == null || e.getKey() == null || aliases.contains(e.getKey())) {
// e.getKey() (alias) can be null in case of constant expressions. see
// input8.q
ExprWalkerInfo extractPushdownPreds = ExprWalkerProcFactory
@@ -627,25 +644,12 @@ public final class OpProcFactory {
}
ewi.merge(extractPushdownPreds);
logExpr(nd, extractPushdownPreds);
- } else {
- hasUnpushedPredicates = true;
}
}
owi.putPrunedPreds((Operator<? extends OperatorDesc>) nd, ewi);
return hasUnpushedPredicates;
}
- private void removeCandidates(Operator<?> operator, OpWalkerInfo owi) {
- if (operator instanceof FilterOperator) {
- owi.getCandidateFilterOps().remove(operator);
- }
- if (operator.getChildOperators() != null) {
- for (Operator<?> child : operator.getChildOperators()) {
- removeCandidates(child, owi);
- }
- }
- }
-
protected ExprWalkerInfo mergeChildrenPred(Node nd, OpWalkerInfo owi,
Set<String> excludedAliases, boolean ignoreAliases)
throws SemanticException {
@@ -678,13 +682,16 @@ public final class OpProcFactory {
|| pushDownPreds.getFinalCandidates().size() == 0) {
return null;
}
+ return createFilter(op, pushDownPreds.getFinalCandidates(), owi);
+ }
+ protected static Object createFilter(Operator op,
+ Map<String, List<ExprNodeDesc>> predicates, OpWalkerInfo owi) {
RowResolver inputRR = owi.getRowResolver(op);
// combine all predicates into a single expression
List<ExprNodeDesc> preds = new ArrayList<ExprNodeDesc>();
- Iterator<List<ExprNodeDesc>> iterator = pushDownPreds.getFinalCandidates()
- .values().iterator();
+ Iterator<List<ExprNodeDesc>> iterator = predicates.values().iterator();
while (iterator.hasNext()) {
for (ExprNodeDesc pred : iterator.next()) {
preds = ExprNodeDescUtils.split(pred, preds);
@@ -831,8 +838,8 @@ public final class OpProcFactory {
}
if (decomposed.residualPredicate != null) {
LOG.debug(
- "Residual predicate: "
- + decomposed.residualPredicate.getExprString());
+ "Residual predicate: "
+ + decomposed.residualPredicate.getExprString());
}
}
tableScanDesc.setFilterExpr(decomposed.pushedPredicate);
@@ -847,10 +854,6 @@ public final class OpProcFactory {
return new JoinPPD();
}
- public static NodeProcessor getRSProc() {
- return new ReduceSinkPPD();
- }
-
public static NodeProcessor getTSProc() {
return new TableScanPPD();
}
@@ -871,12 +874,16 @@ public final class OpProcFactory {
return new ScriptPPD();
}
+ public static NodeProcessor getLVFProc() {
+ return new LateralViewForwardPPD();
+ }
+
public static NodeProcessor getUDTFProc() {
return new UDTFPPD();
}
- public static NodeProcessor getLVFProc() {
- return new LateralViewForwardPPD();
+ public static NodeProcessor getLVJProc() {
+ return new JoinerPPD();
}
private OpProcFactory() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicatePushDown.java Sat Mar 22 05:01:42 2014
@@ -24,9 +24,9 @@ import java.util.Map;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
+import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
import org.apache.hadoop.hive.ql.exec.LimitOperator;
import org.apache.hadoop.hive.ql.exec.PTFOperator;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.ScriptOperator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.UDTFOperator;
@@ -91,18 +91,15 @@ public class PredicatePushDown implement
FilterOperator.getOperatorName() + "%"),
OpProcFactory.getFilterProc());
opRules.put(new RuleRegExp("R2",
- PTFOperator.getOperatorName() + "%"),
- OpProcFactory.getPTFProc());
+ PTFOperator.getOperatorName() + "%"),
+ OpProcFactory.getPTFProc());
opRules.put(new RuleRegExp("R3",
CommonJoinOperator.getOperatorName() + "%"),
OpProcFactory.getJoinProc());
opRules.put(new RuleRegExp("R4",
- ReduceSinkOperator.getOperatorName() + "%"),
- OpProcFactory.getRSProc());
- opRules.put(new RuleRegExp("R5",
TableScanOperator.getOperatorName() + "%"),
OpProcFactory.getTSProc());
- opRules.put(new RuleRegExp("R6",
+ opRules.put(new RuleRegExp("R5",
ScriptOperator.getOperatorName() + "%"),
OpProcFactory.getSCRProc());
opRules.put(new RuleRegExp("R6",
@@ -114,6 +111,9 @@ public class PredicatePushDown implement
opRules.put(new RuleRegExp("R8",
LateralViewForwardOperator.getOperatorName() + "%"),
OpProcFactory.getLVFProc());
+ opRules.put(new RuleRegExp("R9",
+ LateralViewJoinOperator.getOperatorName() + "%"),
+ OpProcFactory.getLVJProc());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Modified: hive/trunk/ql/src/test/queries/clientpositive/lateral_view_ppd.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/lateral_view_ppd.q?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/lateral_view_ppd.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/lateral_view_ppd.q Sat Mar 22 05:01:42 2014
@@ -11,3 +11,7 @@ SELECT value, myCol FROM (SELECT * FROM
EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE key='0';
SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol LATERAL VIEW explode(array(1,2,3)) myTable2 AS myCol2) a WHERE key='0';
+
+-- HIVE-4293 Predicates following UDTF operator are removed by PPD
+EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0';
+SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0';
\ No newline at end of file
Added: hive/trunk/ql/src/test/queries/clientpositive/ppd_join4.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ppd_join4.q?rev=1580168&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ppd_join4.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/ppd_join4.q Sat Mar 22 05:01:42 2014
@@ -0,0 +1,22 @@
+create table dual(a string);
+
+set hive.optimize.ppd=true;
+drop table if exists test_tbl ;
+
+create table test_tbl (id string,name string);
+
+insert into table test_tbl
+select 'a','b' from dual;
+
+explain
+select t2.*
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a';
+
+select t2.*
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a';
\ No newline at end of file
Modified: hive/trunk/ql/src/test/queries/clientpositive/ppd_transform.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ppd_transform.q?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ppd_transform.q (original)
+++ hive/trunk/ql/src/test/queries/clientpositive/ppd_transform.q Sat Mar 22 05:01:42 2014
@@ -36,3 +36,12 @@ FROM (
CLUSTER BY tkey
) tmap
SELECT tmap.tkey, tmap.tvalue WHERE tmap.tkey < 100;
+
+-- test described in HIVE-4598
+
+EXPLAIN
+FROM (
+ FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y
+) reduced
+insert overwrite local directory '/tmp/a' select * where x='a' or x='b'
+insert overwrite local directory '/tmp/b' select * where x='c' or x='d';
Added: hive/trunk/ql/src/test/queries/clientpositive/ppd_udtf.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/ppd_udtf.q?rev=1580168&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/ppd_udtf.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/ppd_udtf.q Sat Mar 22 05:01:42 2014
@@ -0,0 +1,12 @@
+explain
+SELECT value from (
+ select explode(array(key, value)) as (value) from (
+ select * FROM src WHERE key > 400
+ ) A
+) B WHERE value < 450;
+
+SELECT value from (
+ select explode(array(key, value)) as (value) from (
+ select * FROM src WHERE key > 400
+ ) A
+) B WHERE value < 450;
Modified: hive/trunk/ql/src/test/results/clientpositive/cluster.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/cluster.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/cluster.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/cluster.q.out Sat Mar 22 05:01:42 2014
@@ -393,33 +393,33 @@ STAGE PLANS:
TableScan
alias: x
Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col0 (type: string), _col1 (type: string)
+ Filter Operator
+ predicate: (key = 20) (type: boolean)
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
Reduce Operator Tree:
Extract
- Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col0 = 20) (type: boolean)
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: string)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ctas_colname.q.out Sat Mar 22 05:01:42 2014
@@ -1135,15 +1135,15 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 < 9) and _col1 is not null) (type: boolean)
- Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ predicate: _col1 is not null (type: boolean)
+ Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col0 (type: string)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
Modified: hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out Sat Mar 22 05:01:42 2014
@@ -100,26 +100,23 @@ STAGE PLANS:
Lateral View Forward
Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: key (type: string), value (type: string)
- outputColumnNames: key, value
+ expressions: value (type: string)
+ outputColumnNames: value
Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col4
- Statistics: Num rows: 28 Data size: 5610 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((_col0 = '0') and (_col4 = 1)) (type: boolean)
- Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col1 (type: string), _col4 (type: int)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ outputColumnNames: _col1, _col4
+ Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Select Operator
expressions: array(1,2,3) (type: array<int>)
outputColumnNames: _col0
@@ -127,19 +124,19 @@ STAGE PLANS:
UDTF Operator
Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
function name: explode
- Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col4
- Statistics: Num rows: 28 Data size: 5610 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: ((_col0 = '0') and (_col4 = 1)) (type: boolean)
- Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (col = 1) (type: boolean)
+ Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col1, _col4
+ Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col1 (type: string), _col4 (type: int)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 7 Data size: 1402 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 21 Data size: 4207 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
@@ -408,3 +405,86 @@ val_0 2
val_0 3
val_0 3
val_0 3
+PREHOOK: query: -- HIVE-4293 Predicates following UDTF operator are removed by PPD
+EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+PREHOOK: type: QUERY
+POSTHOOK: query: -- HIVE-4293 Predicates following UDTF operator are removed by PPD
+EXPLAIN SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key = '0') (type: boolean)
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Forward
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: value (type: string)
+ outputColumnNames: value
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col1, _col4
+ Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Select Operator
+ expressions: array(1,2,3) (type: array<int>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
+ function name: explode
+ Filter Operator
+ predicate: (col > 1) (type: boolean)
+ Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col1, _col4
+ Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: string), _col4 (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 18 Data size: 3606 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value, myCol FROM (SELECT * FROM src LATERAL VIEW explode(array(1,2,3)) myTable AS myCol WHERE myCol > 1) a WHERE key='0'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+val_0 2
+val_0 3
+val_0 2
+val_0 3
+val_0 2
+val_0 3
Modified: hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ppd2.q.out Sat Mar 22 05:01:42 2014
@@ -60,24 +60,20 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1
Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 14 Data size: 2805 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: (_col1 > 1) (type: boolean)
+ Filter Operator
+ predicate: (_col1 > 1) (type: boolean)
+ Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint)
+ outputColumnNames: _col0, _col1
Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col0 (type: string), _col1 (type: bigint)
- outputColumnNames: _col0, _col1
+ File Output Operator
+ compressed: false
Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
- Statistics: Num rows: 4 Data size: 801 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
Modified: hive/trunk/ql/src/test/results/clientpositive/ppd_gby.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppd_gby.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppd_gby.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ppd_gby.q.out Sat Mar 22 05:01:42 2014
@@ -263,7 +263,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > 'val_200') and ((_col1 > 30) or (_col0 < 'val_400'))) (type: boolean)
+ predicate: ((_col1 > 30) or (_col0 < 'val_400')) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string)
Modified: hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ppd_gby2.q.out Sat Mar 22 05:01:42 2014
@@ -181,7 +181,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((_col0 > 'val_200') and ((_col1 > 30) or (_col0 < 'val_400'))) (type: boolean)
+ predicate: ((_col1 > 30) or (_col0 < 'val_400')) (type: boolean)
Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
Added: hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out?rev=1580168&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/ppd_join4.q.out Sat Mar 22 05:01:42 2014
@@ -0,0 +1,144 @@
+PREHOOK: query: create table dual(a string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table dual(a string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dual
+PREHOOK: query: drop table if exists test_tbl
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists test_tbl
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table test_tbl (id string,name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table test_tbl (id string,name string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test_tbl
+PREHOOK: query: insert into table test_tbl
+select 'a','b' from dual
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dual
+PREHOOK: Output: default@test_tbl
+POSTHOOK: query: insert into table test_tbl
+select 'a','b' from dual
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dual
+POSTHOOK: Output: default@test_tbl
+POSTHOOK: Lineage: test_tbl.id SIMPLE []
+POSTHOOK: Lineage: test_tbl.name SIMPLE []
+PREHOOK: query: explain
+select t2.*
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select t2.*
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: test_tbl.id SIMPLE []
+POSTHOOK: Lineage: test_tbl.name SIMPLE []
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-1 depends on stages: Stage-2
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: test_tbl
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Filter Operator
+ predicate: ((name = 'c') and (id = 'a')) (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: id (type: string), name (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
+ Reduce Operator Tree:
+ Extract
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: t3
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Filter Operator
+ predicate: (id = 'a') (type: boolean)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Reduce Output Operator
+ key expressions: id (type: string)
+ sort order: +
+ Map-reduce partition columns: id (type: string)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ value expressions: _col0 (type: string), _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {VALUE._col0} {VALUE._col1}
+ 1
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: select t2.*
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test_tbl
+#### A masked pattern was here ####
+POSTHOOK: query: select t2.*
+from
+(select id,name from (select id,name from test_tbl) t1 sort by id) t2
+join test_tbl t3 on (t2.id=t3.id )
+where t2.name='c' and t3.id='a'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test_tbl
+#### A masked pattern was here ####
+POSTHOOK: Lineage: test_tbl.id SIMPLE []
+POSTHOOK: Lineage: test_tbl.name SIMPLE []
Modified: hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/ppd_transform.q.out Sat Mar 22 05:01:42 2014
@@ -342,3 +342,83 @@ POSTHOOK: Input: default@src
97 val_97
98 val_98
98 val_98
+PREHOOK: query: -- test described in HIVE-4598
+
+EXPLAIN
+FROM (
+ FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y
+) reduced
+#### A masked pattern was here ####
+PREHOOK: type: QUERY
+POSTHOOK: query: -- test described in HIVE-4598
+
+EXPLAIN
+FROM (
+ FROM ( SELECT * FROM src ) mapout REDUCE * USING 'cat' AS x,y
+) reduced
+#### A masked pattern was here ####
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-2 is a root stage
+ Stage-0 depends on stages: Stage-2
+ Stage-1 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Transform Operator
+ command: cat
+ output info:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: ((_col0 = 'a') or (_col0 = 'b')) (type: boolean)
+ Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Filter Operator
+ predicate: ((_col0 = 'c') or (_col0 = 'd')) (type: boolean)
+ Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 28 Data size: 5611 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
+ Stage: Stage-1
+ Move Operator
+ files:
+ hdfs directory: false
+#### A masked pattern was here ####
+
Added: hive/trunk/ql/src/test/results/clientpositive/ppd_udtf.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/ppd_udtf.q.out?rev=1580168&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/ppd_udtf.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/ppd_udtf.q.out Sat Mar 22 05:01:42 2014
@@ -0,0 +1,125 @@
+PREHOOK: query: explain
+SELECT value from (
+ select explode(array(key, value)) as (value) from (
+ select * FROM src WHERE key > 400
+ ) A
+) B WHERE value < 450
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT value from (
+ select explode(array(key, value)) as (value) from (
+ select * FROM src WHERE key > 400
+ ) A
+) B WHERE value < 450
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 29 Data size: 5812 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key > 400) (type: boolean)
+ Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: array(key,value) (type: array<string>)
+ outputColumnNames: _col0
+ Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
+ UDTF Operator
+ Statistics: Num rows: 9 Data size: 1803 Basic stats: COMPLETE Column stats: NONE
+ function name: explode
+ Filter Operator
+ predicate: (col < 450) (type: boolean)
+ Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: col (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 601 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+PREHOOK: query: SELECT value from (
+ select explode(array(key, value)) as (value) from (
+ select * FROM src WHERE key > 400
+ ) A
+) B WHERE value < 450
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT value from (
+ select explode(array(key, value)) as (value) from (
+ select * FROM src WHERE key > 400
+ ) A
+) B WHERE value < 450
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+409
+401
+406
+429
+403
+417
+430
+446
+413
+417
+439
+438
+427
+437
+430
+404
+411
+431
+418
+404
+436
+409
+413
+419
+435
+401
+402
+430
+449
+401
+438
+432
+403
+406
+409
+406
+401
+424
+431
+431
+424
+438
+414
+439
+417
+444
+429
+443
+406
+401
+421
+407
+448
+414
+403
Modified: hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/subquery_notin_having.q.out Sat Mar 22 05:01:42 2014
@@ -440,7 +440,7 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 14 Data size: 1531 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col2 - _col1) > 600) and (_col0 is null or _col1 is null)) (type: boolean)
+ predicate: ((_col0 is null or _col1 is null) and ((_col2 - _col1) > 600)) (type: boolean)
Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
Select Operator
Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
@@ -598,7 +598,7 @@ STAGE PLANS:
TableScan
Reduce Output Operator
sort order:
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Reduce Operator Tree:
Join Operator
condition map:
@@ -689,10 +689,10 @@ STAGE PLANS:
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 7 Data size: 765 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((_col1 - _col2) > 600) and _col0 is null) (type: boolean)
- Statistics: Num rows: 1 Data size: 109 Basic stats: COMPLETE Column stats: NONE
+ predicate: ((_col1 - _col2) > 600) (type: boolean)
+ Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
Select Operator
- Statistics: Num rows: 1 Data size: 109 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 2 Data size: 218 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
mode: hash
@@ -721,16 +721,16 @@ STAGE PLANS:
Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
Filter Operator
predicate: (_col0 = 0) (type: boolean)
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Select Operator
expressions: _col0 (type: bigint)
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
Group By Operator
keys: _col0 (type: bigint)
mode: hash
outputColumnNames: _col0
- Statistics: Num rows: 1 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE
File Output Operator
compressed: false
table:
Modified: hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out?rev=1580168&r1=1580167&r2=1580168&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out Sat Mar 22 05:01:42 2014
@@ -303,26 +303,23 @@ STAGE PLANS:
Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
Lateral View Join Operator
outputColumnNames: _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 12 Data size: 472 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col4 is not null (type: boolean)
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
- Select Operator
- expressions: _col5 (type: string)
- outputColumnNames: _col5
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
- Group By Operator
- aggregations: count()
- keys: _col5 (type: string)
- mode: hash
- outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
- Reduce Output Operator
- key expressions: _col0 (type: string)
- sort order: +
- Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
- value expressions: _col1 (type: bigint)
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col5 (type: string)
+ outputColumnNames: _col5
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ keys: _col5 (type: string)
+ mode: hash
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: bigint)
Select Operator
expressions: jstring (type: string), 'f1' (type: string), 'f2' (type: string), 'f3' (type: string), 'f4' (type: string), 'f5' (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
@@ -330,27 +327,27 @@ STAGE PLANS:
UDTF Operator
Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
function name: json_tuple
- Lateral View Join Operator
- outputColumnNames: _col4, _col5, _col6, _col7, _col8
- Statistics: Num rows: 12 Data size: 472 Basic stats: COMPLETE Column stats: NONE
- Filter Operator
- predicate: _col4 is not null (type: boolean)
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: c0 is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+ Lateral View Join Operator
+ outputColumnNames: _col4, _col5, _col6, _col7, _col8
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col5 (type: string)
outputColumnNames: _col5
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
Group By Operator
aggregations: count()
keys: _col5 (type: string)
mode: hash
outputColumnNames: _col0, _col1
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
Map-reduce partition columns: _col0 (type: string)
- Statistics: Num rows: 6 Data size: 236 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 9 Data size: 354 Basic stats: COMPLETE Column stats: NONE
value expressions: _col1 (type: bigint)
Reduce Operator Tree:
Group By Operator
@@ -358,11 +355,11 @@ STAGE PLANS:
keys: KEY._col0 (type: string)
mode: mergepartial
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: _col0 (type: string), _col1 (type: bigint)
outputColumnNames: _col0, _col1
- Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
table:
@@ -377,14 +374,14 @@ STAGE PLANS:
Reduce Output Operator
key expressions: _col0 (type: string)
sort order: +
- Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
value expressions: _col0 (type: string), _col1 (type: bigint)
Reduce Operator Tree:
Extract
- Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
File Output Operator
compressed: false
- Statistics: Num rows: 3 Data size: 118 Basic stats: COMPLETE Column stats: NONE
+ Statistics: Num rows: 4 Data size: 157 Basic stats: COMPLETE Column stats: NONE
table:
input format: org.apache.hadoop.mapred.TextInputFormat
output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat