You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2012/08/24 14:03:43 UTC
svn commit: r1376891 - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/
java/org/apache/hadoop/hive/ql/optimizer/ test/queries/clientpositive/
test/results/clientnegative/ test/results/clientpositive/
Author: namit
Date: Fri Aug 24 12:03:42 2012
New Revision: 1376891
URL: http://svn.apache.org/viewvc?rev=1376891&view=rev
Log:
HIVE-3226 ColumnPruner is not working on LateralView
(Navis via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/lateral_view_cp.q
hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hive/trunk/ql/src/test/results/clientnegative/udf_assert_true.q.out
hive/trunk/ql/src/test/results/clientnegative/udf_assert_true2.q.out
hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out
hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out
hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out
hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/LateralViewJoinOperator.java Fri Aug 24 12:03:42 2012
@@ -78,8 +78,8 @@ public class LateralViewJoinOperator ext
// The expected tags from the parent operators. See processOp() before
// changing the tags.
- static final int SELECT_TAG = 0;
- static final int UDTF_TAG = 1;
+ public static final int SELECT_TAG = 0;
+ public static final int UDTF_TAG = 1;
@Override
protected void initializeOp(Configuration hconf) throws HiveException {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java Fri Aug 24 12:03:42 2012
@@ -94,6 +94,8 @@ public class ColumnPruner implements Tra
.getTableScanProc());
opRules.put(new RuleRegExp("R8", "LVJ%"), ColumnPrunerProcFactory
.getLateralViewJoinProc());
+ opRules.put(new RuleRegExp("R9", "LVF%"), ColumnPrunerProcFactory
+ .getLateralViewForwardProc());
// The dispatcher fires the processor corresponding to the closest matching
// rule and passes the context along
Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcCtx.java Fri Aug 24 12:03:42 2012
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.ql.exec.Se
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.parse.OpParseContext;
+import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -155,4 +156,18 @@ public class ColumnPrunerProcCtx impleme
return cols;
}
+
+ /**
+ * Create the list of internal columns for select tag of LV
+ */
+ public List<String> getSelectColsFromLVJoin(RowResolver rr,
+ List<String> colList) throws SemanticException {
+ List<String> columns = new ArrayList<String>();
+ for (String col : colList) {
+ if (rr.reverseLookup(col) != null) {
+ columns.add(col);
+ }
+ }
+ return columns;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Fri Aug 24 12:03:42 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.exec.Fi
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.LateralViewForwardOperator;
import org.apache.hadoop.hive.ql.exec.LateralViewJoinOperator;
import org.apache.hadoop.hive.ql.exec.LimitOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
@@ -56,6 +57,7 @@ import org.apache.hadoop.hive.ql.parse.O
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
+import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
@@ -314,9 +316,8 @@ public final class ColumnPrunerProcFacto
Object... nodeOutputs) throws SemanticException {
LateralViewJoinOperator op = (LateralViewJoinOperator) nd;
ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
- List<String> cols = new ArrayList<String>();
+ List<String> cols = cppCtx.genColLists(op);
- cols = cppCtx.genColLists(op);
Map<String, ExprNodeDesc> colExprMap = op.getColumnExprMap();
// As columns go down the DAG, the LVJ will transform internal column
@@ -332,6 +333,22 @@ public final class ColumnPrunerProcFacto
colsAfterReplacement.add(col);
}
}
+ // this is SEL(*) cols + UDTF cols
+ List<String> outputCols = op.getConf().getOutputInternalColNames();
+ if (outputCols.size() != cols.size()) {
+ // cause we cannot prune columns from UDTF branch currently, extract
+ // columns from SEL(*) branch only and append all columns from UDTF branch to it
+ ArrayList<String> newColNames = new ArrayList<String>();
+ for (String col : cols) {
+ int index = outputCols.indexOf(col);
+ // colExprMap.size() == size of cols from SEL(*) branch
+ if (index >= 0 && index < colExprMap.size()) {
+ newColNames.add(col);
+ }
+ }
+ newColNames.addAll(outputCols.subList(colExprMap.size(), outputCols.size()));
+ op.getConf().setOutputInternalColNames(newColNames);
+ }
cppCtx.getPrunedColLists().put(op,
colsAfterReplacement);
@@ -340,6 +357,44 @@ public final class ColumnPrunerProcFacto
}
/**
+ * The Node Processor for Column Pruning on Lateral View Forward Operators.
+ */
+ public static class ColumnPrunerLateralViewForwardProc extends ColumnPrunerDefaultProc {
+ @Override
+ public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx ctx,
+ Object... nodeOutputs) throws SemanticException {
+ super.process(nd, stack, ctx, nodeOutputs);
+ LateralViewForwardOperator op = (LateralViewForwardOperator) nd;
+ ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
+
+ // get the SEL(*) branch
+ Operator<?> select = op.getChildOperators().get(LateralViewJoinOperator.SELECT_TAG);
+
+ // these are from ColumnPrunerSelectProc
+ List<String> cols = cppCtx.getPrunedColList(select);
+ RowResolver rr = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
+ if (rr.getColumnInfos().size() != cols.size()) {
+ ArrayList<ExprNodeDesc> colList = new ArrayList<ExprNodeDesc>();
+ ArrayList<String> outputColNames = new ArrayList<String>();
+ for (String col : cols) {
+ // revert output cols of SEL(*) to ExprNodeColumnDesc
+ String[] tabcol = rr.reverseLookup(col);
+ ColumnInfo colInfo = rr.get(tabcol[0], tabcol[1]);
+ ExprNodeColumnDesc colExpr = new ExprNodeColumnDesc(colInfo.getType(),
+ colInfo.getInternalName(), colInfo.getTabAlias(), colInfo.getIsVirtualCol());
+ colList.add(colExpr);
+ outputColNames.add(col);
+ }
+ // replace SEL(*) to SEL(exprs)
+ ((SelectDesc)select.getConf()).setSelStarNoCompute(false);
+ ((SelectDesc)select.getConf()).setColList(colList);
+ ((SelectDesc)select.getConf()).setOutputColumnNames(outputColNames);
+ }
+ return null;
+ }
+ }
+
+ /**
* The Node Processor for Column Pruning on Select Operators.
*/
public static class ColumnPrunerSelectProc implements NodeProcessor {
@@ -347,8 +402,8 @@ public final class ColumnPrunerProcFacto
Object... nodeOutputs) throws SemanticException {
SelectOperator op = (SelectOperator) nd;
ColumnPrunerProcCtx cppCtx = (ColumnPrunerProcCtx) ctx;
- List<String> cols = new ArrayList<String>();
+ LateralViewJoinOperator lvJoin = null;
if (op.getChildOperators() != null) {
for (Operator<? extends Serializable> child : op.getChildOperators()) {
// If one of my children is a FileSink or Script, return all columns.
@@ -364,11 +419,22 @@ public final class ColumnPrunerProcFacto
.put(op, cppCtx.getColsFromSelectExpr(op));
return null;
}
+ if (op.getConf().isSelStarNoCompute() && child instanceof LateralViewJoinOperator) {
+ // this SEL is SEL(*) for LV
+ lvJoin = (LateralViewJoinOperator) child;
+ }
}
}
- cols = cppCtx.genColLists(op);
+ List<String> cols = cppCtx.genColLists(op);
SelectDesc conf = op.getConf();
+
+ if (lvJoin != null) {
+ // get columns for SEL(*) from LVJ
+ RowResolver rr = cppCtx.getOpToParseCtxMap().get(op).getRowResolver();
+ cppCtx.getPrunedColLists().put(op, cppCtx.getSelectColsFromLVJoin(rr, cols));
+ return null;
+ }
// The input to the select does not matter. Go over the expressions
// and return the ones which have a marked column
cppCtx.getPrunedColLists().put(op,
@@ -541,6 +607,10 @@ public final class ColumnPrunerProcFacto
return new ColumnPrunerLateralViewJoinProc();
}
+ public static ColumnPrunerLateralViewForwardProc getLateralViewForwardProc() {
+ return new ColumnPrunerLateralViewForwardProc();
+ }
+
/**
* The Node Processor for Column Pruning on Join Operators.
*/
Added: hive/trunk/ql/src/test/queries/clientpositive/lateral_view_cp.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/lateral_view_cp.q?rev=1376891&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/lateral_view_cp.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/lateral_view_cp.q Fri Aug 24 12:03:42 2012
@@ -0,0 +1,8 @@
+--HIVE 3226
+drop table array_valued_src;
+create table array_valued_src (key string, value array<string>);
+insert overwrite table array_valued_src select key, array(value) from src;
+
+-- replace sel(*) to sel(exprs) for reflecting CP result properly
+explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val;
+select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val;
Modified: hive/trunk/ql/src/test/results/clientnegative/udf_assert_true.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_assert_true.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_assert_true.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_assert_true.q.out Fri Aug 24 12:03:42 2012
@@ -23,9 +23,8 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: assert_true((_col2 > 0))
@@ -46,7 +45,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: assert_true((_col2 > 0))
@@ -95,9 +94,8 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: assert_true((_col2 < 2))
@@ -118,7 +116,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: assert_true((_col2 < 2))
Modified: hive/trunk/ql/src/test/results/clientnegative/udf_assert_true2.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientnegative/udf_assert_true2.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientnegative/udf_assert_true2.q.out (original)
+++ hive/trunk/ql/src/test/results/clientnegative/udf_assert_true2.q.out Fri Aug 24 12:03:42 2012
@@ -18,9 +18,8 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: (1 + assert_true((_col2 < 2)))
@@ -41,7 +40,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: (1 + assert_true((_col2 < 2)))
Modified: hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/lateral_view.q.out Fri Aug 24 12:03:42 2012
@@ -159,9 +159,8 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: _col2
@@ -182,7 +181,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Select Operator
expressions:
expr: _col2
@@ -222,14 +221,16 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: _col2
+ type: int
+ outputColumnNames: _col2
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -252,7 +253,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -275,12 +276,15 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: _col2
+ type: int
+ outputColumnNames: _col2
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -303,7 +307,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -345,14 +349,12 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col3
Select Operator
expressions:
expr: _col3
@@ -373,7 +375,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col3
Select Operator
expressions:
expr: _col3
@@ -394,12 +396,11 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col2
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col3
Select Operator
expressions:
expr: _col3
@@ -420,7 +421,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col3
Select Operator
expressions:
expr: _col3
@@ -522,9 +523,8 @@ STAGE PLANS:
alias: tmp_pyang_lv
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col1
Select Operator
expressions:
expr: _col1
@@ -545,7 +545,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col1
Select Operator
expressions:
expr: _col1
@@ -861,16 +861,17 @@ STAGE PLANS:
outputColumnNames: _col0, _col1
Select Operator
expressions:
- expr: _col1
- type: string
expr: array(_col0)
type: array<string>
- outputColumnNames: _col0, _col1
+ outputColumnNames: _col1
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: _col1
+ type: array<string>
+ outputColumnNames: _col1
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col1, _col2
Select Operator
expressions:
expr: _col1
@@ -892,7 +893,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2
+ outputColumnNames: _col1, _col2
Select Operator
expressions:
expr: _col1
Added: hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out?rev=1376891&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out (added)
+++ hive/trunk/ql/src/test/results/clientpositive/lateral_view_cp.q.out Fri Aug 24 12:03:42 2012
@@ -0,0 +1,177 @@
+PREHOOK: query: --HIVE 3226
+drop table array_valued_src
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: --HIVE 3226
+drop table array_valued_src
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table array_valued_src (key string, value array<string>)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: create table array_valued_src (key string, value array<string>)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@array_valued_src
+PREHOOK: query: insert overwrite table array_valued_src select key, array(value) from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@array_valued_src
+POSTHOOK: query: insert overwrite table array_valued_src select key, array(value) from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@array_valued_src
+POSTHOOK: Lineage: array_valued_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: array_valued_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- replace sel(*) to sel(exprs) for reflecting CP result properly
+explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val
+PREHOOK: type: QUERY
+POSTHOOK: query: -- replace sel(*) to sel(exprs) for reflecting CP result properly
+explain select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val
+POSTHOOK: type: QUERY
+POSTHOOK: Lineage: array_valued_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: array_valued_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_LATERAL_VIEW (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION explode (TOK_TABLE_OR_COL array_val)) val (TOK_TABALIAS c))) (TOK_SUBQUERY (TOK_QUERY (TOK_FROM (TOK_JOIN (TOK_TABREF (TOK_TABNAME src) a) (TOK_TABREF (TOK_TABNAME array_valued_src) b) (= (. (TOK_TABLE_OR_COL a) key) (. (TOK_TABLE_OR_COL b) key)))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (. (TOK_TABLE_OR_COL a) key) key) (TOK_SELEXPR (. (TOK_TABLE_OR_COL b) value) array_val)))) i))) (TOK_INSERT (TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (TOK_FUNCTION count (TOK_TABLE_OR_COL val))))))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ i:a
+ TableScan
+ alias: a
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 0
+ i:b
+ TableScan
+ alias: b
+ Reduce Output Operator
+ key expressions:
+ expr: key
+ type: string
+ sort order: +
+ Map-reduce partition columns:
+ expr: key
+ type: string
+ tag: 1
+ value expressions:
+ expr: value
+ type: array<string>
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0
+ 1 {VALUE._col1}
+ handleSkewJoin: false
+ outputColumnNames: _col5
+ Select Operator
+ expressions:
+ expr: _col5
+ type: array<string>
+ outputColumnNames: _col1
+ Lateral View Forward
+ Select Operator
+ Lateral View Join Operator
+ outputColumnNames: _col2
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ outputColumnNames: _col2
+ Group By Operator
+ aggregations:
+ expr: count(_col2)
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ Select Operator
+ expressions:
+ expr: _col1
+ type: array<string>
+ outputColumnNames: _col0
+ UDTF Operator
+ function name: explode
+ Lateral View Join Operator
+ outputColumnNames: _col2
+ Select Operator
+ expressions:
+ expr: _col2
+ type: string
+ outputColumnNames: _col2
+ Group By Operator
+ aggregations:
+ expr: count(_col2)
+ bucketGroup: false
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+
+ Stage: Stage-2
+ Map Reduce
+ Alias -> Map Operator Tree:
+#### A masked pattern was here ####
+ Reduce Output Operator
+ sort order:
+ tag: -1
+ value expressions:
+ expr: _col0
+ type: bigint
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations:
+ expr: count(VALUE._col0)
+ bucketGroup: false
+ mode: mergepartial
+ outputColumnNames: _col0
+ Select Operator
+ expressions:
+ expr: _col0
+ type: bigint
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+
+
+PREHOOK: query: select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val
+PREHOOK: type: QUERY
+PREHOOK: Input: default@array_valued_src
+PREHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: query: select count(val) from (select a.key as key, b.value as array_val from src a join array_valued_src b on a.key=b.key) i lateral view explode (array_val) c as val
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@array_valued_src
+POSTHOOK: Input: default@src
+#### A masked pattern was here ####
+POSTHOOK: Lineage: array_valued_src.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: array_valued_src.value EXPRESSION [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+1028
Modified: hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/lateral_view_ppd.q.out Fri Aug 24 12:03:42 2012
@@ -226,9 +226,16 @@ STAGE PLANS:
alias: srcpart
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: value
+ type: string
+ expr: ds
+ type: string
+ expr: hr
+ type: string
+ outputColumnNames: value, ds, hr
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ outputColumnNames: _col1, _col2, _col3, _col4
Select Operator
expressions:
expr: _col1
@@ -262,7 +269,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ outputColumnNames: _col1, _col2, _col3, _col4
Select Operator
expressions:
expr: _col1
Modified: hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_json_tuple.q.out Fri Aug 24 12:03:42 2012
@@ -63,9 +63,12 @@ STAGE PLANS:
alias: a
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6
Select Operator
expressions:
expr: _col0
@@ -118,7 +121,7 @@ STAGE PLANS:
UDTF Operator
function name: json_tuple
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6
Select Operator
expressions:
expr: _col0
@@ -297,9 +300,12 @@ STAGE PLANS:
alias: a
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6
Select Operator
expressions:
expr: _col0
@@ -340,7 +346,7 @@ STAGE PLANS:
UDTF Operator
function name: json_tuple
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6
Select Operator
expressions:
expr: _col0
@@ -418,9 +424,8 @@ STAGE PLANS:
alias: a
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6
Filter Operator
predicate:
expr: _col2 is not null
@@ -469,7 +474,7 @@ STAGE PLANS:
UDTF Operator
function name: json_tuple
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6
Filter Operator
predicate:
expr: _col2 is not null
Modified: hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_parse_url_tuple.q.out Fri Aug 24 12:03:42 2012
@@ -84,9 +84,12 @@ STAGE PLANS:
alias: a
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Select Operator
expressions:
expr: _col0
@@ -163,7 +166,7 @@ STAGE PLANS:
UDTF Operator
function name: parse_url_tuple
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Select Operator
expressions:
expr: _col0
@@ -376,9 +379,12 @@ STAGE PLANS:
alias: a
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: key
+ type: string
+ outputColumnNames: key
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Select Operator
expressions:
expr: _col0
@@ -449,7 +455,7 @@ STAGE PLANS:
UDTF Operator
function name: parse_url_tuple
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
+ outputColumnNames: _col0, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13
Select Operator
expressions:
expr: _col0
@@ -542,9 +548,8 @@ STAGE PLANS:
alias: a
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Filter Operator
predicate:
expr: _col10 is not null
@@ -601,7 +606,7 @@ STAGE PLANS:
UDTF Operator
function name: parse_url_tuple
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ outputColumnNames: _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
Filter Operator
predicate:
expr: _col10 is not null
Modified: hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/udtf_stack.q.out Fri Aug 24 12:03:42 2012
@@ -23,9 +23,8 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -54,7 +53,7 @@ STAGE PLANS:
UDTF Operator
function name: stack
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -95,9 +94,8 @@ STAGE PLANS:
alias: src
Lateral View Forward
Select Operator
- SELECT * : (no compute)
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
@@ -128,7 +126,7 @@ STAGE PLANS:
UDTF Operator
function name: stack
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col2, _col3
Select Operator
expressions:
expr: _col2
Modified: hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out?rev=1376891&r1=1376890&r2=1376891&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/union_lateralview.q.out Fri Aug 24 12:03:42 2012
@@ -83,9 +83,14 @@ STAGE PLANS:
Union
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col0, _col1, _col3
Select Operator
expressions:
expr: _col3
@@ -117,7 +122,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col0, _col1, _col3
Select Operator
expressions:
expr: _col3
@@ -156,9 +161,14 @@ STAGE PLANS:
Union
Lateral View Forward
Select Operator
- SELECT * : (no compute)
+ expressions:
+ expr: _col0
+ type: string
+ expr: _col1
+ type: string
+ outputColumnNames: _col0, _col1
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col0, _col1, _col3
Select Operator
expressions:
expr: _col3
@@ -190,7 +200,7 @@ STAGE PLANS:
UDTF Operator
function name: explode
Lateral View Join Operator
- outputColumnNames: _col0, _col1, _col2, _col3
+ outputColumnNames: _col0, _col1, _col3
Select Operator
expressions:
expr: _col3