You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2013/04/12 10:08:54 UTC
svn commit: r1467196 [1/2] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/parse/
test/queries/clientpositive/ test/results/clientpositive/
Author: namit
Date: Fri Apr 12 08:08:53 2013
New Revision: 1467196
URL: http://svn.apache.org/r1467196
Log:
HIVE-4294 Single sourced multi query cannot handle lateral view
(Navis via namit)
Added:
hive/trunk/ql/src/test/queries/clientpositive/multi_insert_lateral_view.q
hive/trunk/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=1467196&r1=1467195&r2=1467196&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Fri Apr 12 08:08:53 2013
@@ -1549,6 +1549,7 @@ public abstract class Operator<T extends
start++;
}
builder.append(name);
+ start += name.length();
if (added) {
if (op.getNumChild() > 0) {
List<Operator<?>> children = op.getChildOperators();
@@ -1559,7 +1560,7 @@ public abstract class Operator<T extends
builder.append(' ');
}
}
- toString(builder, visited, children.get(i), start += name.length());
+ toString(builder, visited, children.get(i), start);
}
}
return true;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1467196&r1=1467195&r2=1467196&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Fri Apr 12 08:08:53 2013
@@ -1828,6 +1828,7 @@ body
:
insertClause
selectClause
+ lateralView?
whereClause?
groupByClause?
havingClause?
@@ -1836,11 +1837,12 @@ body
distributeByClause?
sortByClause?
window_clause?
- limitClause? -> ^(TOK_INSERT insertClause?
- selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
+ limitClause? -> ^(TOK_INSERT insertClause
+ selectClause lateralView? whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
distributeByClause? sortByClause? window_clause? limitClause?)
|
selectClause
+ lateralView?
whereClause?
groupByClause?
havingClause?
@@ -1850,7 +1852,7 @@ body
sortByClause?
window_clause?
limitClause? -> ^(TOK_INSERT ^(TOK_DESTINATION ^(TOK_DIR TOK_TMP_FILE))
- selectClause whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
+ selectClause lateralView? whereClause? groupByClause? havingClause? orderByClause? clusterByClause?
distributeByClause? sortByClause? window_clause? limitClause?)
;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java?rev=1467196&r1=1467195&r2=1467196&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java Fri Apr 12 08:08:53 2013
@@ -90,6 +90,8 @@ public class QBParseInfo {
*/
private final HashMap<String, ArrayList<ASTNode>> aliasToLateralViews;
+ private final HashMap<String, ASTNode> destToLateralView;
+
/* Order by clause */
private final HashMap<String, ASTNode> destToOrderby;
private final HashMap<String, Integer> destToLimit;
@@ -111,6 +113,7 @@ public class QBParseInfo {
nameToDest = new HashMap<String, ASTNode>();
nameToSample = new HashMap<String, TableSample>();
exprToColumnAlias = new HashMap<ASTNode, String>();
+ destToLateralView = new HashMap<String, ASTNode>();
destToSelExpr = new LinkedHashMap<String, ASTNode>();
destToWhereExpr = new HashMap<String, ASTNode>();
destToGroupby = new HashMap<String, ASTNode>();
@@ -552,6 +555,9 @@ public class QBParseInfo {
return nameToSample;
}
+ public HashMap<String, ASTNode> getDestToLateralView() {
+ return destToLateralView;
+ }
protected static enum ClauseType {
CLUSTER_BY_CLAUSE,
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1467196&r1=1467195&r2=1467196&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Fri Apr 12 08:08:53 2013
@@ -1023,7 +1023,13 @@ public class SemanticAnalyzer extends Ba
.getMsg(partition.toString()));
}
}
-
+ skipRecursion = false;
+ break;
+ case HiveParser.TOK_LATERAL_VIEW:
+ // todo: nested LV
+ assert ast.getChildCount() == 1;
+ qb.getParseInfo().getDestToLateralView().put(ctx_1.dest, ast);
+ break;
default:
skipRecursion = false;
break;
@@ -3989,7 +3995,7 @@ public class SemanticAnalyzer extends Ba
}
@SuppressWarnings({"nls"})
- private Operator genGroupByPlan1MRMultiReduceGB(List<String> dests, QB qb, Operator input)
+ private Operator genGroupByPlan1ReduceMultiGBY(List<String> dests, QB qb, Operator input)
throws SemanticException {
QBParseInfo parseInfo = qb.getParseInfo();
@@ -6811,9 +6817,14 @@ public class SemanticAnalyzer extends Ba
// Return the common distinct expression
// There should be more than 1 destination, with group bys in all of them.
private List<ASTNode> getCommonDistinctExprs(QB qb, Operator input) {
- RowResolver inputRR = opParseCtx.get(input).getRowResolver();
QBParseInfo qbp = qb.getParseInfo();
+ // If a grouping set aggregation is present, common processing is not possible
+ if (!qbp.getDestCubes().isEmpty() || !qbp.getDestRollups().isEmpty()
+ || !qbp.getDestToLateralView().isEmpty()) {
+ return null;
+ }
+ RowResolver inputRR = opParseCtx.get(input).getRowResolver();
TreeSet<String> ks = new TreeSet<String>();
ks.addAll(qbp.getClauseNames());
@@ -6822,15 +6833,10 @@ public class SemanticAnalyzer extends Ba
return null;
}
- List<ExprNodeDesc.ExprNodeDescEqualityWrapper> oldList = null;
+ List<ExprNodeDesc> oldList = null;
List<ASTNode> oldASTList = null;
for (String dest : ks) {
- // If a grouping set aggregation is present, common processing is not possible
- if (!qbp.getDestCubes().isEmpty() || !qbp.getDestRollups().isEmpty()) {
- return null;
- }
-
// If a filter is present, common processing is not possible
if (qbp.getWhrForClause(dest) != null) {
return null;
@@ -6847,7 +6853,7 @@ public class SemanticAnalyzer extends Ba
return null;
}
- List<ExprNodeDesc.ExprNodeDescEqualityWrapper> currDestList;
+ List<ExprNodeDesc> currDestList;
try {
currDestList = getDistinctExprs(qbp, dest, inputRR);
} catch (SemanticException e) {
@@ -6968,10 +6974,9 @@ public class SemanticAnalyzer extends Ba
// Groups the clause names into lists so that any two clauses in the same list has the same
// group by and distinct keys and no clause appears in more than one list. Returns a list of the
// lists of clauses.
- private List<List<String>> getCommonGroupByDestGroups(QB qb, Operator input)
- throws SemanticException {
+ private List<List<String>> getCommonGroupByDestGroups(QB qb,
+ Map<String, Operator<? extends OperatorDesc>> inputs) throws SemanticException {
- RowResolver inputRR = opParseCtx.get(input).getRowResolver();
QBParseInfo qbp = qb.getParseInfo();
TreeSet<String> ks = new TreeSet<String>();
@@ -6989,29 +6994,31 @@ public class SemanticAnalyzer extends Ba
return commonGroupByDestGroups;
}
- List<List<ExprNodeDesc.ExprNodeDescEqualityWrapper>> sprayKeyLists =
- new ArrayList<List<ExprNodeDesc.ExprNodeDescEqualityWrapper>>(ks.size());
+ List<Operator<? extends OperatorDesc>> inputOperators =
+ new ArrayList<Operator<? extends OperatorDesc>>(ks.size());
+ List<List<ExprNodeDesc>> sprayKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size());
// Iterate over each clause
for (String dest : ks) {
-
- List<ExprNodeDesc.ExprNodeDescEqualityWrapper> sprayKeys =
- getDistinctExprs(qbp, dest, inputRR);
+ Operator input = inputs.get(dest);
+ RowResolver inputRR = opParseCtx.get(input).getRowResolver();
+ List<ExprNodeDesc> sprayKeys = getDistinctExprs(qbp, dest, inputRR);
// Add the group by expressions
List<ASTNode> grpByExprs = getGroupByForClause(qbp, dest);
for (ASTNode grpByExpr : grpByExprs) {
- ExprNodeDesc.ExprNodeDescEqualityWrapper grpByExprWrapper =
- new ExprNodeDesc.ExprNodeDescEqualityWrapper(genExprNodeDesc(grpByExpr, inputRR));
- if (!sprayKeys.contains(grpByExprWrapper)) {
- sprayKeys.add(grpByExprWrapper);
+ ExprNodeDesc exprDesc = genExprNodeDesc(grpByExpr, inputRR);
+ if (ExprNodeDescUtils.indexOf(exprDesc, sprayKeys) < 0) {
+ sprayKeys.add(exprDesc);
}
}
// Loop through each of the lists of exprs, looking for a match
boolean found = false;
for (int i = 0; i < sprayKeyLists.size(); i++) {
-
+ if (!input.equals(inputOperators.get(i))) {
+ continue;
+ }
if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) {
continue;
}
@@ -7024,6 +7031,7 @@ public class SemanticAnalyzer extends Ba
// No match was found, so create new entries
if (!found) {
+ inputOperators.add(input);
sprayKeyLists.add(sprayKeys);
List<String> destGroup = new ArrayList<String>();
destGroup.add(dest);
@@ -7035,15 +7043,13 @@ public class SemanticAnalyzer extends Ba
}
// Returns whether or not two lists contain the same elements independent of order
- private boolean matchExprLists(List<ExprNodeDesc.ExprNodeDescEqualityWrapper> list1,
- List<ExprNodeDesc.ExprNodeDescEqualityWrapper> list2) {
+ private boolean matchExprLists(List<ExprNodeDesc> list1, List<ExprNodeDesc> list2) {
if (list1.size() != list2.size()) {
return false;
}
-
- for (ExprNodeDesc.ExprNodeDescEqualityWrapper exprNodeDesc : list1) {
- if (!list2.contains(exprNodeDesc)) {
+ for (ExprNodeDesc exprNodeDesc : list1) {
+ if (ExprNodeDescUtils.indexOf(exprNodeDesc, list2) < 0) {
return false;
}
}
@@ -7051,23 +7057,20 @@ public class SemanticAnalyzer extends Ba
return true;
}
- // Returns a list of the distinct exprs for a given clause name as
- // ExprNodeDesc.ExprNodeDescEqualityWrapper without duplicates
- private List<ExprNodeDesc.ExprNodeDescEqualityWrapper>
- getDistinctExprs(QBParseInfo qbp, String dest, RowResolver inputRR) throws SemanticException {
+ // Returns a list of the distinct exprs without duplicates for a given clause name
+ private List<ExprNodeDesc> getDistinctExprs(QBParseInfo qbp, String dest, RowResolver inputRR)
+ throws SemanticException {
List<ASTNode> distinctAggExprs = qbp.getDistinctFuncExprsForClause(dest);
- List<ExprNodeDesc.ExprNodeDescEqualityWrapper> distinctExprs =
- new ArrayList<ExprNodeDesc.ExprNodeDescEqualityWrapper>();
+ List<ExprNodeDesc> distinctExprs = new ArrayList<ExprNodeDesc>();
for (ASTNode distinctAggExpr : distinctAggExprs) {
// 0 is function name
for (int i = 1; i < distinctAggExpr.getChildCount(); i++) {
ASTNode parameter = (ASTNode) distinctAggExpr.getChild(i);
- ExprNodeDesc.ExprNodeDescEqualityWrapper distinctExpr =
- new ExprNodeDesc.ExprNodeDescEqualityWrapper(genExprNodeDesc(parameter, inputRR));
- if (!distinctExprs.contains(distinctExpr)) {
- distinctExprs.add(distinctExpr);
+ ExprNodeDesc expr = genExprNodeDesc(parameter, inputRR);
+ if (ExprNodeDescUtils.indexOf(expr, distinctExprs) < 0) {
+ distinctExprs.add(expr);
}
}
}
@@ -7096,6 +7099,7 @@ public class SemanticAnalyzer extends Ba
QBParseInfo qbp = qb.getParseInfo();
TreeSet<String> ks = new TreeSet<String>(qbp.getClauseNames());
+ Map<String, Operator<? extends OperatorDesc>> inputs = createInputForDests(qb, input, ks);
// For multi-group by with the same distinct, we ignore all user hints
// currently. It doesnt matter whether he has asked to do
// map-side aggregation or not. Map side aggregation is turned off
@@ -7148,7 +7152,7 @@ public class SemanticAnalyzer extends Ba
// expressions, otherwise treat all the expressions as a single group
if (conf.getBoolVar(HiveConf.ConfVars.HIVEMULTIGROUPBYSINGLEREDUCER)) {
try {
- commonGroupByDestGroups = getCommonGroupByDestGroups(qb, curr);
+ commonGroupByDestGroups = getCommonGroupByDestGroups(qb, inputs);
} catch (SemanticException e) {
LOG.error("Failed to group clauses by common spray keys.", e);
}
@@ -7168,6 +7172,8 @@ public class SemanticAnalyzer extends Ba
}
String firstDest = commonGroupByDestGroup.get(0);
+ input = inputs.get(firstDest);
+
// Constructs a standard group by plan if:
// There is no other subquery with the same group by/distinct keys or
// (There are no aggregations in a representative query for the group and
@@ -7182,7 +7188,7 @@ public class SemanticAnalyzer extends Ba
// Go over all the destination tables
for (String dest : commonGroupByDestGroup) {
- curr = input;
+ curr = inputs.get(dest);
if (qbp.getWhrForClause(dest) != null) {
curr = genFilterPlan(dest, qb, curr);
@@ -7215,7 +7221,7 @@ public class SemanticAnalyzer extends Ba
curr = genPostGroupByBodyPlan(curr, dest, qb);
}
} else {
- curr = genGroupByPlan1MRMultiReduceGB(commonGroupByDestGroup, qb, input);
+ curr = genGroupByPlan1ReduceMultiGBY(commonGroupByDestGroup, qb, input);
}
}
}
@@ -7228,6 +7234,16 @@ public class SemanticAnalyzer extends Ba
return curr;
}
+ private Map<String, Operator<? extends OperatorDesc>> createInputForDests(QB qb,
+ Operator<? extends OperatorDesc> input, Set<String> dests) throws SemanticException {
+ Map<String, Operator<? extends OperatorDesc>> inputs =
+ new HashMap<String, Operator<? extends OperatorDesc>>();
+ for (String dest : dests) {
+ inputs.put(dest, genLateralViewPlanForDest(dest, qb, input));
+ }
+ return inputs;
+ }
+
private Operator genPostGroupByBodyPlan(Operator curr, String dest, QB qb)
throws SemanticException {
@@ -8037,71 +8053,7 @@ public class SemanticAnalyzer extends Ba
// -> LateralViewJoinOperator
//
- RowResolver lvForwardRR = new RowResolver();
- RowResolver source = opParseCtx.get(op).getRowResolver();
- for (ColumnInfo col : source.getColumnInfos()) {
- if (col.getIsVirtualCol() && col.isHiddenVirtualCol()) {
- continue;
- }
- String[] tabCol = source.reverseLookup(col.getInternalName());
- lvForwardRR.put(tabCol[0], tabCol[1], col);
- }
-
- Operator lvForward = putOpInsertMap(OperatorFactory.getAndMakeChild(
- new LateralViewForwardDesc(), new RowSchema(lvForwardRR.getColumnInfos()),
- op), lvForwardRR);
-
- // The order in which the two paths are added is important. The
- // lateral view join operator depends on having the select operator
- // give it the row first.
-
- // Get the all path by making a select(*).
- RowResolver allPathRR = opParseCtx.get(lvForward).getRowResolver();
- // Operator allPath = op;
- Operator allPath = putOpInsertMap(OperatorFactory.getAndMakeChild(
- new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()),
- lvForward), allPathRR);
- // Get the UDTF Path
- QB blankQb = new QB(null, null, false);
- Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
- .getChild(0), blankQb, lvForward);
- // add udtf aliases to QB
- for (String udtfAlias : blankQb.getAliases()) {
- qb.addAlias(udtfAlias);
- }
- RowResolver udtfPathRR = opParseCtx.get(udtfPath).getRowResolver();
-
- // Merge the two into the lateral view join
- // The cols of the merged result will be the combination of both the
- // cols of the UDTF path and the cols of the all path. The internal
- // names have to be changed to avoid conflicts
-
- RowResolver lateralViewRR = new RowResolver();
- ArrayList<String> outputInternalColNames = new ArrayList<String>();
-
- LVmergeRowResolvers(allPathRR, lateralViewRR, outputInternalColNames);
- LVmergeRowResolvers(udtfPathRR, lateralViewRR, outputInternalColNames);
-
- // For PPD, we need a column to expression map so that during the walk,
- // the processor knows how to transform the internal col names.
- // Following steps are dependant on the fact that we called
- // LVmerge.. in the above order
- Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
-
- int i = 0;
- for (ColumnInfo c : allPathRR.getColumnInfos()) {
- String internalName = getColumnInternalName(i);
- i++;
- colExprMap.put(internalName,
- new ExprNodeColumnDesc(c.getType(), c.getInternalName(),
- c.getTabAlias(), c.getIsVirtualCol()));
- }
-
- Operator lateralViewJoin = putOpInsertMap(OperatorFactory
- .getAndMakeChild(new LateralViewJoinDesc(outputInternalColNames),
- new RowSchema(lateralViewRR.getColumnInfos()), allPath,
- udtfPath), lateralViewRR);
- lateralViewJoin.setColumnExprMap(colExprMap);
+ Operator lateralViewJoin = genLateralViewPlan(qb, op, lateralViewTree);
op = lateralViewJoin;
}
e.setValue(op);
@@ -8109,6 +8061,85 @@ public class SemanticAnalyzer extends Ba
}
}
+ private Operator genLateralViewPlanForDest(String dest, QB qb, Operator op)
+ throws SemanticException {
+ ASTNode lateralViewTree = qb.getParseInfo().getDestToLateralView().get(dest);
+ if (lateralViewTree != null) {
+ return genLateralViewPlan(qb, op, lateralViewTree);
+ }
+ return op;
+ }
+
+ private Operator genLateralViewPlan(QB qb, Operator op, ASTNode lateralViewTree)
+ throws SemanticException {
+ RowResolver lvForwardRR = new RowResolver();
+ RowResolver source = opParseCtx.get(op).getRowResolver();
+ for (ColumnInfo col : source.getColumnInfos()) {
+ if (col.getIsVirtualCol() && col.isHiddenVirtualCol()) {
+ continue;
+ }
+ String[] tabCol = source.reverseLookup(col.getInternalName());
+ lvForwardRR.put(tabCol[0], tabCol[1], col);
+ }
+
+ Operator lvForward = putOpInsertMap(OperatorFactory.getAndMakeChild(
+ new LateralViewForwardDesc(), new RowSchema(lvForwardRR.getColumnInfos()),
+ op), lvForwardRR);
+
+ // The order in which the two paths are added is important. The
+ // lateral view join operator depends on having the select operator
+ // give it the row first.
+
+ // Get the all path by making a select(*).
+ RowResolver allPathRR = opParseCtx.get(lvForward).getRowResolver();
+ // Operator allPath = op;
+ Operator allPath = putOpInsertMap(OperatorFactory.getAndMakeChild(
+ new SelectDesc(true), new RowSchema(allPathRR.getColumnInfos()),
+ lvForward), allPathRR);
+ // Get the UDTF Path
+ QB blankQb = new QB(null, null, false);
+ Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
+ .getChild(0), blankQb, lvForward);
+ // add udtf aliases to QB
+ for (String udtfAlias : blankQb.getAliases()) {
+ qb.addAlias(udtfAlias);
+ }
+ RowResolver udtfPathRR = opParseCtx.get(udtfPath).getRowResolver();
+
+ // Merge the two into the lateral view join
+ // The cols of the merged result will be the combination of both the
+ // cols of the UDTF path and the cols of the all path. The internal
+ // names have to be changed to avoid conflicts
+
+ RowResolver lateralViewRR = new RowResolver();
+ ArrayList<String> outputInternalColNames = new ArrayList<String>();
+
+ LVmergeRowResolvers(allPathRR, lateralViewRR, outputInternalColNames);
+ LVmergeRowResolvers(udtfPathRR, lateralViewRR, outputInternalColNames);
+
+ // For PPD, we need a column to expression map so that during the walk,
+ // the processor knows how to transform the internal col names.
+ // Following steps are dependant on the fact that we called
+ // LVmerge.. in the above order
+ Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
+
+ int i = 0;
+ for (ColumnInfo c : allPathRR.getColumnInfos()) {
+ String internalName = getColumnInternalName(i);
+ i++;
+ colExprMap.put(internalName,
+ new ExprNodeColumnDesc(c.getType(), c.getInternalName(),
+ c.getTabAlias(), c.getIsVirtualCol()));
+ }
+
+ Operator lateralViewJoin = putOpInsertMap(OperatorFactory
+ .getAndMakeChild(new LateralViewJoinDesc(outputInternalColNames),
+ new RowSchema(lateralViewRR.getColumnInfos()), allPath,
+ udtfPath), lateralViewRR);
+ lateralViewJoin.setColumnExprMap(colExprMap);
+ return lateralViewJoin;
+ }
+
/**
* A helper function that gets all the columns and respective aliases in the
* source and puts them into dest. It renames the internal names of the
@@ -8704,7 +8735,7 @@ public class SemanticAnalyzer extends Ba
}
if (LOG.isDebugEnabled()) {
- LOG.debug("\n" + Operator.toString(pCtx.getTopOps().values()));
+ LOG.debug("Before logical optimization\n" + Operator.toString(pCtx.getTopOps().values()));
}
Optimizer optm = new Optimizer();
@@ -8713,7 +8744,7 @@ public class SemanticAnalyzer extends Ba
pCtx = optm.optimize();
if (LOG.isDebugEnabled()) {
- LOG.debug("\n" + Operator.toString(pCtx.getTopOps().values()));
+ LOG.debug("After logical optimization\n" + Operator.toString(pCtx.getTopOps().values()));
}
// Generate column access stats if required - wait until column pruning takes place
Added: hive/trunk/ql/src/test/queries/clientpositive/multi_insert_lateral_view.q
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/multi_insert_lateral_view.q?rev=1467196&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/multi_insert_lateral_view.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/multi_insert_lateral_view.q Fri Apr 12 08:08:53 2013
@@ -0,0 +1,102 @@
+create table src_10 as select * from src limit 10;
+
+create table src_lv1 (key string, value string);
+create table src_lv2 (key string, value string);
+create table src_lv3 (key string, value string);
+
+-- 2LV
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-FS[12]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[13]-FS[14]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+explain
+from src_10
+insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C
+insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C;
+
+from src_10
+insert overwrite table src_lv1 select key, C lateral view explode(array(key+1, key+2)) A as C
+insert overwrite table src_lv2 select key, C lateral view explode(array(key+3, key+4)) A as C;
+
+select * from src_lv1;
+select * from src_lv2;
+
+-- 2(LV+GBY)
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key;
+
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(C) lateral view explode(array(key+3, key+4)) A as C group by key;
+
+select * from src_lv1;
+select * from src_lv2;
+
+-- (LV+GBY) + RS:2GBY
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[6]-GBY[7]-RS[8]-GBY[9]-SEL[10]-FS[11]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -FIL[12]-SEL[13]-RS[14]-FOR[15]-FIL[16]-GBY[17]-SEL[18]-FS[19]
+-- -FIL[20]-GBY[21]-SEL[22]-FS[23]
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, count(value) where key > 200 group by key
+insert overwrite table src_lv3 select key, count(value) where key < 200 group by key;
+
+from src_10
+insert overwrite table src_lv1 select key, sum(C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, count(value) where key > 200 group by key
+insert overwrite table src_lv3 select key, count(value) where key < 200 group by key;
+
+select * from src_lv1;
+select * from src_lv2;
+select * from src_lv3;
+
+-- todo: shared distinct columns (should work with hive.optimize.multigroupby.common.distincts)
+-- 2(LV+GBY) + RS:2GBY
+-- TS[0]-LVF[1]-SEL[2]-LVJ[5]-SEL[11]-GBY[12]-RS[13]-GBY[14]-SEL[15]-FS[16]
+-- -SEL[3]-UDTF[4]-LVJ[5]
+-- -LVF[6]-SEL[7]-LVJ[10]-SEL[17]-GBY[18]-RS[19]-GBY[20]-SEL[21]-FS[22]
+-- -SEL[8]-UDTF[9]-LVJ[10]
+-- -SEL[23]-GBY[24]-RS[25]-GBY[26]-SEL[27]-FS[28]
+explain
+from src_10
+insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C
+insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C
+insert overwrite table src_lv3 select value, sum(distinct key) group by value;
+
+from src_10
+insert overwrite table src_lv1 select C, sum(distinct key) lateral view explode(array(key+1, key+2)) A as C group by C
+insert overwrite table src_lv2 select C, sum(distinct key) lateral view explode(array(key+3, key+4)) A as C group by C
+insert overwrite table src_lv3 select value, sum(distinct key) group by value;
+
+select * from src_lv1;
+select * from src_lv2;
+select * from src_lv3;
+
+create table src_lv4 (key string, value string);
+
+-- Common distincts optimization works across non-lateral view queries, but not across lateral view multi inserts
+explain
+from src_10
+insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key
+insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value
+insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value;
+
+from src_10
+insert overwrite table src_lv1 select key, sum(distinct C) lateral view explode(array(key+1, key+2)) A as C group by key
+insert overwrite table src_lv2 select key, sum(distinct C) lateral view explode(array(key+3, key+4)) A as C group by key
+insert overwrite table src_lv3 select value, sum(distinct key) where key > 200 group by value
+insert overwrite table src_lv4 select value, sum(distinct key) where key < 200 group by value;
+
+select * from src_lv1;
+select * from src_lv2;
+select * from src_lv3;
+select * from src_lv4;