You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/09/12 03:21:29 UTC
svn commit: r1522098 [19/30] - in /hive/branches/vectorization: ./
beeline/src/test/org/apache/hive/beeline/src/test/ bin/ bin/ext/
common/src/java/org/apache/hadoop/hive/common/
common/src/java/org/apache/hadoop/hive/conf/ conf/ contrib/src/java/org/a...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java Thu Sep 12 01:21:10 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.optimi
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -38,9 +39,10 @@ import org.apache.hadoop.hive.ql.parse.S
*/
public class TableSizeBasedBigTableSelectorForAutoSMJ extends SizeBasedBigTableSelectorForAutoSMJ
implements BigTableSelectorForAutoSMJ {
- public int getBigTablePosition(ParseContext parseCtx, JoinOperator joinOp)
+ public int getBigTablePosition(ParseContext parseCtx, JoinOperator joinOp,
+ Set<Integer> bigTableCandidates)
throws SemanticException {
- int bigTablePos = 0;
+ int bigTablePos = -1;
long maxSize = -1;
HiveConf conf = parseCtx.getConf();
@@ -49,9 +51,15 @@ implements BigTableSelectorForAutoSMJ {
getListTopOps(joinOp, topOps);
int currentPos = 0;
for (TableScanOperator topOp : topOps) {
+
if (topOp == null) {
return -1;
}
+
+ if (!bigTableCandidates.contains(currentPos)) {
+ currentPos++;
+ continue;
+ }
Table table = parseCtx.getTopToTable().get(topOp);
long currentSize = 0;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/CorrelationOptimizer.java Thu Sep 12 01:21:10 2013
@@ -166,7 +166,7 @@ public class CorrelationOptimizer implem
JoinDesc joinDesc = joinOp.getConf();
Byte[] order = joinDesc.getTagOrder();
int numAliases = order.length;
- HashSet<Integer> bigTableCandidates =
+ Set<Integer> bigTableCandidates =
MapJoinProcessor.getBigTableCandidates(joinDesc.getConds());
if (bigTableCandidates == null) {
continue;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/correlation/ReduceSinkDeDuplication.java Thu Sep 12 01:21:10 2013
@@ -27,6 +27,7 @@ import java.util.List;
import java.util.Map;
import java.util.Stack;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExtractOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
@@ -230,8 +231,13 @@ public class ReduceSinkDeDuplication imp
}
/**
- * Current RSDedup remove/replace child RS. So always copies
+ * Current RSDedup remove/replace child RS. For key columns,
+ * sorting order, and the number of reducers, copy
* more specific part of configurations of child RS to that of parent RS.
+ * For partitioning columns, if both child RS and parent RS have been assigned
+ * partitioning columns, we will choose the more general partitioning columns.
+ * If parent RS has not been assigned any partitioning column, we will use
+ * partitioning columns (if exist) of child RS.
*/
protected boolean merge(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer)
throws SemanticException {
@@ -239,20 +245,57 @@ public class ReduceSinkDeDuplication imp
if (result == null) {
return false;
}
+
if (result[0] > 0) {
- ArrayList<ExprNodeDesc> childKCs = cRS.getConf().getKeyCols();
+ // The sorting columns of the child RS are more specific than
+ // those of the parent RS. Assign sorting columns of the child RS
+ // to the parent RS.
+ List<ExprNodeDesc> childKCs = cRS.getConf().getKeyCols();
pRS.getConf().setKeyCols(ExprNodeDescUtils.backtrack(childKCs, cRS, pRS));
}
- if (result[1] > 0) {
- ArrayList<ExprNodeDesc> childPCs = cRS.getConf().getPartitionCols();
- pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS));
+
+ if (result[1] < 0) {
+ // The partitioning columns of the parent RS are more specific than
+ // those of the child RS.
+ List<ExprNodeDesc> childPCs = cRS.getConf().getPartitionCols();
+ if (childPCs != null && !childPCs.isEmpty()) {
+ // If partitioning columns of the child RS are assigned,
+ // assign these to the partitioning columns of the parent RS.
+ pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS));
+ }
+ } else if (result[1] > 0) {
+ // The partitioning columns of the child RS are more specific than
+ // those of the parent RS.
+ List<ExprNodeDesc> parentPCs = pRS.getConf().getPartitionCols();
+ if (parentPCs == null || parentPCs.isEmpty()) {
+ // If partitioning columns of the parent RS are not assigned,
+ // assign partitioning columns of the child RS to the parent RS.
+ ArrayList<ExprNodeDesc> childPCs = cRS.getConf().getPartitionCols();
+ pRS.getConf().setPartitionCols(ExprNodeDescUtils.backtrack(childPCs, cRS, pRS));
+ }
}
+
if (result[2] > 0) {
+ // The sorting order of the child RS is more specific than
+ // that of the parent RS. Assign the sorting order of the child RS
+ // to the parent RS.
+ if (result[0] <= 0) {
+ // Sorting columns of the parent RS are more specific than those of the
+ // child RS but Sorting order of the child RS is more specific than
+ // that of the parent RS.
+ throw new SemanticException("Sorting columns and order don't match. " +
+ "Try set " + HiveConf.ConfVars.HIVEOPTREDUCEDEDUPLICATION + "=false;");
+ }
pRS.getConf().setOrder(cRS.getConf().getOrder());
}
+
if (result[3] > 0) {
+ // The number of reducers of the child RS is more specific than
+ // that of the parent RS. Assign the number of reducers of the child RS
+ // to the parent RS.
pRS.getConf().setNumReducers(cRS.getConf().getNumReducers());
}
+
return true;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/listbucketingpruner/ListBucketingPruner.java Thu Sep 12 01:21:10 2013
@@ -27,7 +27,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.optimizer.PrunerUtils;
@@ -289,13 +288,12 @@ public class ListBucketingPruner impleme
List<List<String>> uniqSkewedValues) throws SemanticException {
// For each entry in dynamic-multi-dimension collection.
List<String> skewedCols = part.getSkewedColNames(); // Retrieve skewed column.
- Map<SkewedValueList, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
+ Map<List<String>, String> mappings = part.getSkewedColValueLocationMaps(); // Retrieve skewed
// map.
assert ListBucketingPrunerUtils.isListBucketingPart(part) : part.getName()
+ " skewed metadata is corrupted. No skewed column and/or location mappings information.";
List<List<String>> skewedValues = part.getSkewedColValues();
List<Boolean> nonSkewedValueMatchResult = new ArrayList<Boolean>();
- SkewedValueList skewedValueList = new SkewedValueList();
for (List<String> cell : collections) {
// Walk through the tree to decide value.
// Example: skewed column: C1, C2 ;
@@ -309,9 +307,8 @@ public class ListBucketingPruner impleme
/* It's valid case if a partition: */
/* 1. is defined with skewed columns and skewed values in metadata */
/* 2. doesn't have all skewed values within its data */
- skewedValueList.setSkewedValueList(cell);
- if (mappings.get(skewedValueList) != null) {
- selectedPaths.add(new Path(mappings.get(skewedValueList)));
+ if (mappings.get(cell) != null) {
+ selectedPaths.add(new Path(mappings.get(cell)));
}
}
} else {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java Thu Sep 12 01:21:10 2013
@@ -36,19 +36,19 @@ import org.apache.hadoop.hive.ql.plan.Op
*/
public class BucketingSortingCtx implements NodeProcessorCtx {
- boolean disableBucketing;
+ private boolean disableBucketing;
// A mapping from an operator to the columns by which it's output is bucketed
- Map<Operator<? extends OperatorDesc>, List<BucketCol>> bucketedColsByOp;
+ private Map<Operator<? extends OperatorDesc>, List<BucketCol>> bucketedColsByOp;
// A mapping from a directory which a FileSinkOperator writes into to the columns by which that
// output is bucketed
- Map<String, List<BucketCol>> bucketedColsByDirectory;
+ private Map<String, List<BucketCol>> bucketedColsByDirectory;
// A mapping from an operator to the columns by which it's output is sorted
- Map<Operator<? extends OperatorDesc>, List<SortCol>> sortedColsByOp;
+ private Map<Operator<? extends OperatorDesc>, List<SortCol>> sortedColsByOp;
// A mapping from a directory which a FileSinkOperator writes into to the columns by which that
// output is sorted
- Map<String, List<SortCol>> sortedColsByDirectory;
+ private Map<String, List<SortCol>> sortedColsByDirectory;
public BucketingSortingCtx(boolean disableBucketing) {
this.disableBucketing = disableBucketing;
@@ -189,13 +189,18 @@ public class BucketingSortingCtx impleme
* to be constant for all equivalent columns.
*/
public static final class SortCol implements BucketSortCol, Serializable {
+
+ public SortCol() {
+ super();
+ }
+
private static final long serialVersionUID = 1L;
// Equivalent aliases for the column
- private final List<String> names = new ArrayList<String>();
+ private List<String> names = new ArrayList<String>();
// Indexes of those equivalent columns
- private final List<Integer> indexes = new ArrayList<Integer>();
+ private List<Integer> indexes = new ArrayList<Integer>();
// Sort order (+|-)
- private final char sortOrder;
+ private char sortOrder;
public SortCol(String name, int index, char sortOrder) {
this(sortOrder);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java Thu Sep 12 01:21:10 2013
@@ -17,16 +17,14 @@
*/
package org.apache.hadoop.hive.ql.optimizer.physical;
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.common.ObjectPair;
@@ -417,7 +415,7 @@ public class CommonJoinTaskDispatcher ex
long aliasTotalKnownInputSize =
getTotalKnownInputSize(context, currWork, pathToAliases, aliasToSize);
- HashSet<Integer> bigTableCandidates = MapJoinProcessor.getBigTableCandidates(joinDesc
+ Set<Integer> bigTableCandidates = MapJoinProcessor.getBigTableCandidates(joinDesc
.getConds());
// no table could be the big table; there is no need to convert
@@ -502,9 +500,7 @@ public class CommonJoinTaskDispatcher ex
}
// deep copy a new mapred work from xml
// Once HIVE-4396 is in, it would be faster to use a cheaper method to clone the plan
- String xml = currTask.getWork().toXML();
- InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8"));
- MapredWork newWork = Utilities.deserializeObject(in);
+ MapredWork newWork = Utilities.clonePlan(currTask.getWork());
// create map join task and set big table as i
ObjectPair<MapRedTask, String> newTaskAlias = convertTaskToMapJoinTask(newWork, i);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java Thu Sep 12 01:21:10 2013
@@ -18,9 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer.physical;
-import java.io.ByteArrayInputStream;
import java.io.Serializable;
-import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -241,16 +239,7 @@ public final class GenMRSkewJoinProcesso
HiveConf.ConfVars.HIVE_MAPPER_CANNOT_SPAN_MULTIPLE_PARTITIONS);
newPlan.setMapperCannotSpanPartns(mapperCannotSpanPartns);
- MapredWork clonePlan = null;
- try {
- String xmlPlan = currPlan.toXML();
- StringBuilder sb = new StringBuilder(xmlPlan);
- ByteArrayInputStream bis;
- bis = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));
- clonePlan = Utilities.deserializeObject(bis);
- } catch (UnsupportedEncodingException e) {
- throw new SemanticException(e);
- }
+ MapredWork clonePlan = Utilities.clonePlan(currPlan);
Operator<? extends OperatorDesc>[] parentOps = new TableScanOperator[tags.length];
for (int k = 0; k < tags.length; k++) {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/LocalMapJoinProcFactory.java Thu Sep 12 01:21:10 2013
@@ -163,7 +163,8 @@ public final class LocalMapJoinProcFacto
if (parent instanceof TableScanOperator) {
tbl = ((TableScanOperator) parent).getTableDesc();
} else {
- throw new SemanticException();
+ throw new SemanticException("Expected parent operator of type TableScanOperator." +
+ "Found " + parent.getClass().getName() + " instead.");
}
} else {
// get parent schema
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java Thu Sep 12 01:21:10 2013
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hive.ql.optimizer.physical;
-import java.io.ByteArrayInputStream;
-import java.io.InputStream;
import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
@@ -148,11 +146,8 @@ public class SortMergeJoinTaskDispatcher
private MapredWork convertSMBWorkToJoinWork(MapredWork currWork, SMBMapJoinOperator oldSMBJoinOp)
throws SemanticException {
try {
- String xml = currWork.toXML();
-
// deep copy a new mapred work
- InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8"));
- MapredWork currJoinWork = Utilities.deserializeObject(in);
+ MapredWork currJoinWork = Utilities.clonePlan(currWork);
SMBMapJoinOperator newSMBJoinOp = getSMBMapJoinOp(currJoinWork);
// Add the row resolver for the new operator
@@ -169,14 +164,13 @@ public class SortMergeJoinTaskDispatcher
}
// create map join task and set big table as bigTablePosition
- private ObjectPair<MapRedTask, String> convertSMBTaskToMapJoinTask(String xml,
+ private ObjectPair<MapRedTask, String> convertSMBTaskToMapJoinTask(MapredWork origWork,
int bigTablePosition,
SMBMapJoinOperator smbJoinOp,
QBJoinTree joinTree)
throws UnsupportedEncodingException, SemanticException {
- // deep copy a new mapred work from xml
- InputStream in = new ByteArrayInputStream(xml.getBytes("UTF-8"));
- MapredWork newWork = Utilities.deserializeObject(in);
+ // deep copy a new mapred work
+ MapredWork newWork = Utilities.clonePlan(origWork);
// create a mapred task for this work
MapRedTask newTask = (MapRedTask) TaskFactory.get(newWork, physicalContext
.getParseContext().getConf());
@@ -240,7 +234,6 @@ public class SortMergeJoinTaskDispatcher
public Task<? extends Serializable> processCurrentTask(MapRedTask currTask,
ConditionalTask conditionalTask, Context context)
throws SemanticException {
-
// whether it contains a sort merge join operator
MapredWork currWork = currTask.getWork();
SMBMapJoinOperator originalSMBJoinOp = getSMBMapJoinOp(currWork);
@@ -284,18 +277,12 @@ public class SortMergeJoinTaskDispatcher
Set<Integer> bigTableCandidates =
MapJoinProcessor.getBigTableCandidates(originalSMBJoinDesc.getConds());
- // no table could be the big table; there is no need to convert
- if (bigTableCandidates == null) {
- return null;
- }
-
HashMap<String, Long> aliasToSize = new HashMap<String, Long>();
Configuration conf = context.getConf();
try {
long aliasTotalKnownInputSize = getTotalKnownInputSize(context, currJoinWork.getMapWork(),
pathToAliases, aliasToSize);
- String xml = currJoinWork.toXML();
long ThresholdOfSmallTblSizeSum = HiveConf.getLongVar(conf,
HiveConf.ConfVars.HIVESMALLTABLESFILESIZE);
@@ -306,8 +293,8 @@ public class SortMergeJoinTaskDispatcher
}
// create map join task for the given big table position
- ObjectPair<MapRedTask, String> newTaskAlias =
- convertSMBTaskToMapJoinTask(xml, bigTablePosition, newSMBJoinOp, joinTree);
+ ObjectPair<MapRedTask, String> newTaskAlias = convertSMBTaskToMapJoinTask(
+ currJoinWork, bigTablePosition, newSMBJoinOp, joinTree);
MapRedTask newTask = newTaskAlias.getFirst();
String bigTableAlias = newTaskAlias.getSecond();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcCtx.java Thu Sep 12 01:21:10 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
/**
* The processor context for partition pruner. This contains the table alias
* that is being currently processed.
+ * TODO: this class may be not useful.
*/
public class ExprProcCtx implements NodeProcessorCtx {
@@ -31,15 +32,8 @@ public class ExprProcCtx implements Node
*/
String tabAlias;
- /**
- * Flag to hold whether there are any non partition columns accessed in the
- * expression.
- */
- boolean hasNonPartCols;
-
public ExprProcCtx(String tabAlias) {
this.tabAlias = tabAlias;
- hasNonPartCols = false;
}
public String getTabAlias() {
@@ -49,12 +43,4 @@ public class ExprProcCtx implements Node
public void setTabAlias(String tabAlias) {
this.tabAlias = tabAlias;
}
-
- public boolean getHasNonPartCols() {
- return hasNonPartCols;
- }
-
- public void setHasNonPartCols(boolean val) {
- hasNonPartCols = val;
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/ExprProcFactory.java Thu Sep 12 01:21:10 2013
@@ -64,7 +64,6 @@ public final class ExprProcFactory exten
newcd = cd.clone();
} else {
newcd = new ExprNodeConstantDesc(cd.getTypeInfo(), null);
- epc.setHasNonPartCols(true);
}
return newcd;
}
@@ -88,12 +87,10 @@ public final class ExprProcFactory exten
* @param pred
* The predicate from which the partition pruner needs to be
* generated
- * @return hasNonPartCols returns true/false depending upon whether this pred
- * has a non partition column
- * @throws SemanticException
+ * @return The pruner expression.
*/
- public static ExprNodeDesc genPruner(String tabAlias, ExprNodeDesc pred,
- boolean hasNonPartCols) throws SemanticException {
+ public static ExprNodeDesc genPruner(
+ String tabAlias, ExprNodeDesc pred) throws SemanticException {
// Create the walker, the rules dispatcher and the context.
ExprProcCtx pprCtx = new ExprProcCtx(tabAlias);
@@ -101,8 +98,6 @@ public final class ExprProcFactory exten
Map<Node, Object> outputMap = PrunerUtils.walkExprTree(pred, pprCtx, getColumnProcessor(),
getFieldProcessor(), getGenericFuncProcessor(), getDefaultExprProcessor());
- hasNonPartCols = pprCtx.getHasNonPartCols();
-
// Get the exprNodeDesc corresponding to the first start node;
return (ExprNodeDesc) outputMap.get(pred);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpProcFactory.java Thu Sep 12 01:21:10 2013
@@ -59,10 +59,7 @@ public final class OpProcFactory extends
String alias = top.getConf().getAlias();
// Generate the partition pruning predicate
- boolean hasNonPartCols = false;
- ExprNodeDesc ppr_pred = ExprProcFactory.genPruner(alias, predicate,
- hasNonPartCols);
- owc.addHasNonPartCols(hasNonPartCols);
+ ExprNodeDesc ppr_pred = ExprProcFactory.genPruner(alias, predicate);
// Add the pruning predicate to the table scan operator
addPruningPred(owc.getOpToPartPruner(), top, ppr_pred);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/OpWalkerCtx.java Thu Sep 12 01:21:10 2013
@@ -26,11 +26,10 @@ import org.apache.hadoop.hive.ql.plan.Ex
/**
* Context class for operator tree walker for partition pruner.
+ * TODO: this class may be not useful.
*/
public class OpWalkerCtx implements NodeProcessorCtx {
- private boolean hasNonPartCols;
-
/**
* Map from tablescan operator to partition pruning predicate that is
* initialized from the ParseContext.
@@ -42,18 +41,9 @@ public class OpWalkerCtx implements Node
*/
public OpWalkerCtx(HashMap<TableScanOperator, ExprNodeDesc> opToPartPruner) {
this.opToPartPruner = opToPartPruner;
- hasNonPartCols = false;
}
public HashMap<TableScanOperator, ExprNodeDesc> getOpToPartPruner() {
return opToPartPruner;
}
-
- public void addHasNonPartCols(boolean val) {
- hasNonPartCols = (hasNonPartCols || val);
- }
-
- public boolean getHasNonPartCols() {
- return hasNonPartCols;
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java Thu Sep 12 01:21:10 2013
@@ -19,11 +19,11 @@
package org.apache.hadoop.hive.ql.optimizer.ppr;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -83,9 +83,8 @@ public class PartExprEvalUtils {
.getPrimitiveJavaObject(evaluateResultO);
}
- static synchronized public Map<PrimitiveObjectInspector, ExprNodeEvaluator> prepareExpr(
- ExprNodeDesc expr, List<String> partNames, List<VirtualColumn> vcs) throws HiveException {
- boolean hasVC = vcs != null && !vcs.isEmpty();
+ static synchronized public ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> prepareExpr(
+ ExprNodeDesc expr, List<String> partNames) throws HiveException {
// Create the row object
List<ObjectInspector> partObjectInspectors = new ArrayList<ObjectInspector>();
for (int i = 0; i < partNames.size(); i++) {
@@ -94,33 +93,14 @@ public class PartExprEvalUtils {
StructObjectInspector objectInspector = ObjectInspectorFactory
.getStandardStructObjectInspector(partNames, partObjectInspectors);
- if (hasVC) {
- List<StructObjectInspector> ois = new ArrayList<StructObjectInspector>(2);
- ois.add(objectInspector);
- ois.add(VirtualColumn.getVCSObjectInspector(vcs));
- objectInspector = ObjectInspectorFactory.getUnionStructObjectInspector(ois);
- }
-
ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(expr);
ObjectInspector evaluateResultOI = evaluator.initialize(objectInspector);
-
- Map<PrimitiveObjectInspector, ExprNodeEvaluator> result =
- new HashMap<PrimitiveObjectInspector, ExprNodeEvaluator>();
- result.put((PrimitiveObjectInspector)evaluateResultOI, evaluator);
- return result;
+ return ObjectPair.create((PrimitiveObjectInspector)evaluateResultOI, evaluator);
}
static synchronized public Object evaluateExprOnPart(
- Map<PrimitiveObjectInspector, ExprNodeEvaluator> pair, Object rowWithPart)
- throws HiveException {
- assert(pair.size() > 0);
- // only get the 1st entry from the map
- Map.Entry<PrimitiveObjectInspector, ExprNodeEvaluator> entry = pair.entrySet().iterator().next();
- PrimitiveObjectInspector evaluateResultOI = entry.getKey();
- ExprNodeEvaluator evaluator = entry.getValue();
-
- Object evaluateResultO = evaluator.evaluate(rowWithPart);
-
- return evaluateResultOI.getPrimitiveJavaObject(evaluateResultO);
+ ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> pair, Object partColValues)
+ throws HiveException {
+ return pair.getFirst().getPrimitiveJavaObject(pair.getSecond().evaluate(partColValues));
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java Thu Sep 12 01:21:10 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer.ppr;
+import java.util.AbstractSequentialList;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -29,11 +30,11 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.common.ObjectPair;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
-import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -44,7 +45,6 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
-import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.PrunerUtils;
import org.apache.hadoop.hive.ql.optimizer.Transform;
import org.apache.hadoop.hive.ql.parse.ParseContext;
@@ -57,10 +57,7 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPAnd;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPOr;
-import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.thrift.TException;
/**
* The transformation step that does partition pruning.
@@ -88,8 +85,6 @@ public class PartitionPruner implements
/* Move logic to PrunerUtils.walkOperatorTree() so that it can be reused. */
PrunerUtils.walkOperatorTree(pctx, opWalkerCtx, OpProcFactory.getFilterProc(),
OpProcFactory.getDefaultProc());
- pctx.setHasNonPartCols(opWalkerCtx.getHasNonPartCols());
-
return pctx;
}
@@ -139,9 +134,8 @@ public class PartitionPruner implements
*/
public static PrunedPartitionList prune(TableScanOperator ts, ParseContext parseCtx,
String alias) throws HiveException {
- return prune(parseCtx.getTopToTable().get(ts),
- parseCtx.getOpToPartPruner().get(ts), parseCtx.getConf(), alias,
- ts.getConf().getVirtualCols(), parseCtx.getPrunedPartitions());
+ return prune(parseCtx.getTopToTable().get(ts), parseCtx.getOpToPartPruner().get(ts),
+ parseCtx.getConf(), alias, parseCtx.getPrunedPartitions());
}
/**
@@ -156,8 +150,6 @@ public class PartitionPruner implements
* for checking whether "strict" mode is on.
* @param alias
* for generating error message only.
- * @param vcs
- * virtual columns referenced
* @param prunedPartitionsMap
* cached result for the table
* @return the partition list for the table that satisfies the partition
@@ -165,8 +157,8 @@ public class PartitionPruner implements
* @throws HiveException
*/
private static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
- HiveConf conf, String alias, List<VirtualColumn> vcs,
- Map<String, PrunedPartitionList> prunedPartitionsMap) throws HiveException {
+ HiveConf conf, String alias, Map<String, PrunedPartitionList> prunedPartitionsMap)
+ throws HiveException {
LOG.trace("Started pruning partiton");
LOG.trace("dbname = " + tab.getDbName());
LOG.trace("tabname = " + tab.getTableName());
@@ -182,13 +174,15 @@ public class PartitionPruner implements
return ret;
}
- ret = getPartitionsFromServer(tab, prunerExpr, vcs, conf, alias);
+ ret = getPartitionsFromServer(tab, prunerExpr, conf, alias);
prunedPartitionsMap.put(key, ret);
return ret;
}
/**
- * Taking a partition pruning expression, remove the null operands.
+ * Taking a partition pruning expression, remove the null operands and non-partition columns.
+ * The reason why there are null operands is ExprProcFactory classes, for example
+ * PPRColumnExprProcessor.
* @param expr original partition pruning expression.
* @return partition pruning expression that only contains partition columns.
*/
@@ -201,19 +195,21 @@ public class PartitionPruner implements
}
} else if (expr instanceof ExprNodeGenericFuncDesc) {
GenericUDF udf = ((ExprNodeGenericFuncDesc)expr).getGenericUDF();
- if (udf instanceof GenericUDFOPAnd ||
- udf instanceof GenericUDFOPOr) {
+ boolean isAnd = udf instanceof GenericUDFOPAnd;
+ if (isAnd || udf instanceof GenericUDFOPOr) {
List<ExprNodeDesc> children = expr.getChildren();
ExprNodeDesc left = children.get(0);
children.set(0, compactExpr(left));
ExprNodeDesc right = children.get(1);
children.set(1, compactExpr(right));
+ // Note that one does not simply compact (not-null or null) to not-null.
+ // Only if we have an "and" is it valid to send one side to metastore.
if (children.get(0) == null && children.get(1) == null) {
return null;
} else if (children.get(0) == null) {
- return children.get(1);
+ return isAnd ? children.get(1) : null;
} else if (children.get(1) == null) {
- return children.get(0);
+ return isAnd ? children.get(0) : null;
}
}
return expr;
@@ -221,13 +217,36 @@ public class PartitionPruner implements
return expr;
}
- private static PrunedPartitionList getPartitionsFromServer(Table tab, ExprNodeDesc prunerExpr,
- List<VirtualColumn> vcs, HiveConf conf, String alias) throws HiveException {
+ /**
+ * See compactExpr. Some things in the expr are replaced with nulls for pruner, however
+ * the virtual columns are not removed (ExprNodeColumnDesc cannot tell them apart from
+ * partition columns), so we do it here.
+ * The expression is only used to prune by partition name, so we have no business with VCs.
+ * @param expr original partition pruning expression.
+ * @param partCols list of partition columns for the table.
+ * @return partition pruning expression that only contains partition columns from the list.
+ */
+ static private ExprNodeDesc removeNonPartCols(ExprNodeDesc expr, List<String> partCols) {
+ if (expr instanceof ExprNodeColumnDesc
+ && !partCols.contains(((ExprNodeColumnDesc) expr).getColumn())) {
+ // Column doesn't appear to be a partition column for the table.
+ return new ExprNodeConstantDesc(expr.getTypeInfo(), null);
+ }
+ if (expr instanceof ExprNodeGenericFuncDesc) {
+ List<ExprNodeDesc> children = expr.getChildren();
+ for (int i = 0; i < children.size(); ++i) {
+ children.set(i, removeNonPartCols(children.get(i), partCols));
+ }
+ }
+ return expr;
+ }
+
+ private static PrunedPartitionList getPartitionsFromServer(Table tab,
+ ExprNodeDesc prunerExpr, HiveConf conf, String alias) throws HiveException {
try {
if (!tab.isPartitioned()) {
// If the table is not partitioned, return everything.
- return new PrunedPartitionList(
- tab, new LinkedHashSet<Partition>(Hive.get().getPartitions(tab)), false);
+ return new PrunedPartitionList(tab, Hive.get().getAllPartitionsForPruner(tab), false);
}
LOG.debug("tabname = " + tab.getTableName() + " is partitioned");
@@ -240,19 +259,19 @@ public class PartitionPruner implements
if (prunerExpr == null) {
// This can happen when hive.mapred.mode=nonstrict and there is no predicates at all.
- return new PrunedPartitionList(tab,
- new LinkedHashSet<Partition>(Hive.get().getPartitions(tab)), false);
+ return new PrunedPartitionList(tab, Hive.get().getAllPartitionsForPruner(tab), false);
}
- // Remove non-partition columns.
+ // Remove virtual columns. See javadoc for details.
+ prunerExpr = removeNonPartCols(prunerExpr, extractPartColNames(tab));
+ // Remove all unknown parts e.g. non-partition columns. See javadoc for details.
ExprNodeDesc compactExpr = compactExpr(prunerExpr.clone());
String oldFilter = prunerExpr.getExprString();
if (compactExpr == null) {
// This could happen when hive.mapred.mode=nonstrict and all the predicates
// are on non-partition columns.
LOG.debug("Filter " + oldFilter + " was null after compacting");
- return new PrunedPartitionList(
- tab, new LinkedHashSet<Partition>(Hive.get().getPartitions(tab)), true);
+ return new PrunedPartitionList(tab, Hive.get().getAllPartitionsForPruner(tab), true);
}
Set<Partition> partitions = new LinkedHashSet<Partition>();
@@ -261,7 +280,7 @@ public class PartitionPruner implements
if (message != null) {
LOG.info(ErrorMsg.INVALID_JDO_FILTER_EXPRESSION.getMsg("by condition '"
+ message + "'"));
- hasUnknownPartitions = pruneBySequentialScan(tab, partitions, prunerExpr, vcs, conf);
+ hasUnknownPartitions = pruneBySequentialScan(tab, partitions, prunerExpr, conf);
} else {
String filter = compactExpr.getExprString();
LOG.debug("Filter w/ compacting: " + filter +"; filter w/o compacting: " + oldFilter);
@@ -282,12 +301,11 @@ public class PartitionPruner implements
* @param tab the table containing the partitions.
* @param partitions the resulting partitions.
* @param prunerExpr the SQL predicate that involves partition columns.
- * @param vcs virtual columns referenced
* @param conf Hive Configuration object, can not be NULL.
* @return true iff the partition pruning expression contains non-partition columns.
*/
static private boolean pruneBySequentialScan(Table tab, Set<Partition> partitions,
- ExprNodeDesc prunerExpr, List<VirtualColumn> vcs, HiveConf conf) throws Exception {
+ ExprNodeDesc prunerExpr, HiveConf conf) throws Exception {
PerfLogger perfLogger = PerfLogger.getPerfLogger();
perfLogger.PerfLogBegin(LOG, PerfLogger.PRUNE_LISTING);
@@ -295,14 +313,10 @@ public class PartitionPruner implements
tab.getDbName(), tab.getTableName(), (short) -1);
String defaultPartitionName = conf.getVar(HiveConf.ConfVars.DEFAULTPARTITIONNAME);
- List<FieldSchema> pCols = tab.getPartCols();
- List<String> partCols = new ArrayList<String>(pCols.size());
- for (FieldSchema pCol : pCols) {
- partCols.add(pCol.getName());
- }
+ List<String> partCols = extractPartColNames(tab);
boolean hasUnknownPartitions = prunePartitionNames(
- partCols, prunerExpr, vcs, defaultPartitionName, partNames);
+ partCols, prunerExpr, defaultPartitionName, partNames);
perfLogger.PerfLogEnd(LOG, PerfLogger.PRUNE_LISTING);
perfLogger.PerfLogBegin(LOG, PerfLogger.PARTITION_RETRIEVING);
@@ -313,41 +327,48 @@ public class PartitionPruner implements
return hasUnknownPartitions;
}
+ private static List<String> extractPartColNames(Table tab) {
+ List<FieldSchema> pCols = tab.getPartCols();
+ List<String> partCols = new ArrayList<String>(pCols.size());
+ for (FieldSchema pCol : pCols) {
+ partCols.add(pCol.getName());
+ }
+ return partCols;
+ }
+
/**
* Prunes partition names to see if they match the prune expression.
- * @param tab Table.
+ * @param columnNames name of partition columns
* @param prunerExpr The expression to match.
- * @param conf Hive configuration.
+ * @param defaultPartitionName name of default partition
* @param partNames Partition names to filter. The list is modified in place.
* @return Whether the list has any partitions for which the expression may or may not match.
*/
public static boolean prunePartitionNames(List<String> columnNames, ExprNodeDesc prunerExpr,
- List<VirtualColumn> vcs, String defaultPartitionName, List<String> partNames)
- throws HiveException, MetaException {
+ String defaultPartitionName, List<String> partNames) throws HiveException, MetaException {
// Prepare the expression to filter on the columns.
- Map<PrimitiveObjectInspector, ExprNodeEvaluator> handle =
- PartExprEvalUtils.prepareExpr(prunerExpr, columnNames, vcs);
+ ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> handle =
+ PartExprEvalUtils.prepareExpr(prunerExpr, columnNames);
- // Filter the name list.
- List<String> values = new ArrayList<String>(columnNames.size());
- Object evalArg = values;
- boolean hasVC = vcs != null && !vcs.isEmpty();
- if (hasVC) {
- Object[] objectWithPart = new Object[2];
- objectWithPart[0] = values;
- evalArg = objectWithPart;
+ // Filter the name list. Removing elements one by one can be slow on e.g. ArrayList,
+ // so let's create a new list and copy it if we don't have a linked list
+ boolean inPlace = partNames instanceof AbstractSequentialList<?>;
+ List<String> partNamesSeq = inPlace ? partNames : new LinkedList<String>(partNames);
+
+ // Array for the values to pass to evaluator.
+ ArrayList<String> values = new ArrayList<String>(columnNames.size());
+ for (int i = 0; i < columnNames.size(); ++i) {
+ values.add(null);
}
boolean hasUnknownPartitions = false;
- Iterator<String> partIter = partNames.iterator();
+ Iterator<String> partIter = partNamesSeq.iterator();
while (partIter.hasNext()) {
String partName = partIter.next();
- LinkedHashMap<String, String> partSpec = Warehouse.makeSpecFromName(partName);
- values.clear();
- values.addAll(partSpec.values());
+ Warehouse.makeValsFromName(partName, values);
// Evaluate the expression tree.
- Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, evalArg);
+ Boolean isNeeded = (Boolean)PartExprEvalUtils.evaluateExprOnPart(handle, values);
boolean isUnknown = (isNeeded == null);
if (!isUnknown && !isNeeded) {
partIter.remove();
@@ -363,6 +384,10 @@ public class PartitionPruner implements
hasUnknownPartitions |= isUnknown;
LOG.debug("retained " + (isUnknown ? "unknown " : "") + "partition: " + partName);
}
+ if (!inPlace) {
+ partNames.clear();
+ partNames.addAll(partNamesSeq);
+ }
return hasUnknownPartitions;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Thu Sep 12 01:21:10 2013
@@ -37,7 +37,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.metastore.api.SkewedValueList;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
@@ -981,31 +980,18 @@ public abstract class BaseSemanticAnalyz
* @return
*/
protected ListBucketingCtx constructListBucketingCtx(List<String> skewedColNames,
- List<List<String>> skewedValues, Map<SkewedValueList, String> skewedColValueLocationMaps,
+ List<List<String>> skewedValues, Map<List<String>, String> skewedColValueLocationMaps,
boolean isStoredAsSubDirectories, HiveConf conf) {
ListBucketingCtx lbCtx = new ListBucketingCtx();
lbCtx.setSkewedColNames(skewedColNames);
lbCtx.setSkewedColValues(skewedValues);
- lbCtx.setLbLocationMap(convertSkewedValueListToSimpleList(skewedColValueLocationMaps));
+ lbCtx.setLbLocationMap(skewedColValueLocationMaps);
lbCtx.setStoredAsSubDirectories(isStoredAsSubDirectories);
lbCtx.setDefaultKey(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_KEY);
lbCtx.setDefaultDirName(ListBucketingPrunerUtils.HIVE_LIST_BUCKETING_DEFAULT_DIR_NAME);
return lbCtx;
}
- // This is done to avoid the need of sending metastore jars to task nodes.
- private Map<List<String>, String> convertSkewedValueListToSimpleList(
- Map<SkewedValueList, String> skewedColValueLocationMaps) {
- if (skewedColValueLocationMaps == null) {
- return null;
- }
- Map<List<String>, String> converted = new HashMap<List<String>, String>();
- for (Map.Entry<SkewedValueList, String> entry : skewedColValueLocationMaps.entrySet()) {
- converted.put(entry.getKey().getSkewedValueList(), entry.getValue());
- }
- return converted;
- }
-
/**
* Given a ASTNode, return list of values.
*
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Thu Sep 12 01:21:10 2013
@@ -125,11 +125,11 @@ lateralView
@init {gParent.msgs.push("lateral view"); }
@after {gParent.msgs.pop(); }
:
- KW_LATERAL KW_VIEW KW_OUTER function tableAlias KW_AS identifier (COMMA identifier)*
- -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW KW_OUTER function tableAlias (KW_AS identifier (COMMA identifier)*)?
+ -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
|
- KW_LATERAL KW_VIEW function tableAlias KW_AS identifier (COMMA identifier)*
- -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW function tableAlias (KW_AS identifier (COMMA identifier)*)?
+ -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier* tableAlias)))
;
tableAlias
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Thu Sep 12 01:21:10 2013
@@ -173,8 +173,9 @@ function
(star=STAR)
| (dist=KW_DISTINCT)? (selectExpression (COMMA selectExpression)*)?
)
- RPAREN -> {$star != null}? ^(TOK_FUNCTIONSTAR functionName)
- -> {$dist == null}? ^(TOK_FUNCTION functionName (selectExpression+)?)
+ RPAREN (KW_OVER ws=window_specification)?
+ -> {$star != null}? ^(TOK_FUNCTIONSTAR functionName $ws?)
+ -> {$dist == null}? ^(TOK_FUNCTION functionName (selectExpression+)? $ws?)
-> ^(TOK_FUNCTIONDI functionName (selectExpression+)?)
;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java Thu Sep 12 01:21:10 2013
@@ -64,6 +64,8 @@ import org.apache.hadoop.hive.serde.serd
*/
public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
+ public static final String METADATA_NAME="_metadata";
+
public ImportSemanticAnalyzer(HiveConf conf) throws SemanticException {
super(conf);
}
@@ -89,7 +91,7 @@ public class ImportSemanticAnalyzer exte
Path fromPath = new Path(fromURI.getScheme(), fromURI.getAuthority(),
fromURI.getPath());
try {
- Path metadataPath = new Path(fromPath, "_metadata");
+ Path metadataPath = new Path(fromPath, METADATA_NAME);
Map.Entry<org.apache.hadoop.hive.metastore.api.Table,
List<Partition>> rv = EximUtil.readMetaData(fs, metadataPath);
dbname = db.getCurrentDatabase();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java Thu Sep 12 01:21:10 2013
@@ -137,14 +137,22 @@ public class MapReduceCompiler {
if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
}
- String cols = loadFileWork.get(0).getColumns();
- String colTypes = loadFileWork.get(0).getColumnTypes();
- String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
- TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
+ LoadFileDesc loadFileDesc = loadFileWork.get(0);
- FetchWork fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(),
+ String cols = loadFileDesc.getColumns();
+ String colTypes = loadFileDesc.getColumnTypes();
+
+ TableDesc resultTab = pCtx.getFetchTabledesc();
+ if (resultTab == null) {
+ String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
+ resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
+ }
+
+ FetchWork fetch = new FetchWork(new Path(loadFileDesc.getSourceDir()).toString(),
resultTab, qb.getParseInfo().getOuterQueryLimit());
+ fetch.setSource(pCtx.getFetchSource());
+ fetch.setSink(pCtx.getFetchSink());
pCtx.setFetchTask((FetchTask) TaskFactory.get(fetch, conf));
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/MetaDataExportListener.java Thu Sep 12 01:21:10 2013
@@ -38,14 +38,13 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.events.PreEventContext;
import org.apache.hadoop.hive.metastore.events.PreEventContext.PreEventType;
import org.apache.hadoop.hive.ql.session.SessionState;
-
+import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
/**
* This class listens for drop events and, if set, exports the table's metadata as JSON to the trash
* of the user performing the drop
*/
public class MetaDataExportListener extends MetaStorePreEventListener {
public static final Log LOG = LogFactory.getLog(MetaDataExportListener.class);
- private static final SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
/** Configure the export listener */
public MetaDataExportListener(Configuration config) {
@@ -65,6 +64,7 @@ public class MetaDataExportListener exte
Path tblPath = new Path(tbl.getSd().getLocation());
fs = wh.getFs(tblPath);
Date now = new Date();
+ SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss");
String dateString = sdf.format(now);
String exportPathString = hiveconf.getVar(HiveConf.ConfVars.METADATA_EXPORT_LOCATION);
boolean moveMetadataToTrash = hiveconf
@@ -83,7 +83,7 @@ public class MetaDataExportListener exte
} catch (IOException e) {
throw new MetaException(e.getMessage());
}
- Path outFile = new Path(metaPath, name + ".metadata");
+ Path outFile = new Path(metaPath, name + ImportSemanticAnalyzer.METADATA_NAME);
try {
SessionState.getConsole().printInfo("Beginning metadata export");
EximUtil.createExportDump(fs, outFile, mTbl, null);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java Thu Sep 12 01:21:10 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.ErrorMs
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
+import org.apache.hadoop.hive.ql.exec.PTFPartition;
import org.apache.hadoop.hive.ql.exec.WindowFunctionInfo;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
@@ -71,7 +72,6 @@ import org.apache.hadoop.hive.ql.plan.PT
import org.apache.hadoop.hive.ql.plan.PTFDesc.RangeBoundaryDef;
import org.apache.hadoop.hive.ql.plan.PTFDesc.ShapeDetails;
import org.apache.hadoop.hive.ql.plan.PTFDesc.ValueBoundaryDef;
-import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowExpressionDef;
import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFrameDef;
import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowFunctionDef;
import org.apache.hadoop.hive.ql.plan.PTFDesc.WindowTableFunctionDef;
@@ -131,6 +131,7 @@ public class PTFTranslator {
init(semAly, hCfg, inputRR, unparseT);
ptfInvocation = qSpec;
ptfDesc = new PTFDesc();
+ ptfDesc.setCfg(hCfg);
ptfDesc.setLlInfo(llInfo);
translatePTFChain();
return ptfDesc;
@@ -143,6 +144,7 @@ public class PTFTranslator {
init(semAly, hCfg, inputRR, unparseT);
windowingSpec = wdwSpec;
ptfDesc = new PTFDesc();
+ ptfDesc.setCfg(hCfg);
ptfDesc.setLlInfo(llInfo);
WindowTableFunctionDef wdwTFnDef = new WindowTableFunctionDef();
ptfDesc.setFuncDef(wdwTFnDef);
@@ -197,83 +199,26 @@ public class PTFTranslator {
/*
* set outputFromWdwFnProcessing
*/
- if (windowFunctions.size() > 0) {
- ArrayList<String> aliases = new ArrayList<String>();
- ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
- for (WindowFunctionDef wFnDef : windowFunctions) {
- aliases.add(wFnDef.getAlias());
- if (wFnDef.isPivotResult()) {
- fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
- } else {
- fieldOIs.add(wFnDef.getOI());
- }
- }
- PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
- StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(
- aliases, fieldOIs);
- tFn.setWdwProcessingOutputOI(wdwOutOI);
- RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef, false);
- ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
- wdwTFnDef.setOutputFromWdwFnProcessing(wdwOutShape);
- }
- else {
- wdwTFnDef.setOutputFromWdwFnProcessing(inpShape);
- }
-
- /*
- * process Wdw expressions
- */
- ShapeDetails wdwOutShape = wdwTFnDef.getOutputFromWdwFnProcessing();
- ArrayList<WindowExpressionDef> windowExpressions = new ArrayList<WindowExpressionDef>();
- if (wdwSpec.getWindowExpressions() != null) {
- for (WindowExpressionSpec expr : wdwSpec.getWindowExpressions()) {
- if (!(expr instanceof WindowFunctionSpec)) {
- try {
- PTFExpressionDef eDef = buildExpressionDef(wdwOutShape, expr.getExpression());
- WindowExpressionDef wdwEDef = new WindowExpressionDef(eDef);
- wdwEDef.setAlias(expr.getAlias());
- windowExpressions.add(wdwEDef);
- } catch (HiveException he) {
- throw new SemanticException(he);
- }
- }
+ ArrayList<String> aliases = new ArrayList<String>();
+ ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
+ for (WindowFunctionDef wFnDef : windowFunctions) {
+ aliases.add(wFnDef.getAlias());
+ if (wFnDef.isPivotResult()) {
+ fieldOIs.add(((ListObjectInspector) wFnDef.getOI()).getListElementObjectInspector());
+ } else {
+ fieldOIs.add(wFnDef.getOI());
}
- wdwTFnDef.setWindowExpressions(windowExpressions);
- }
-
- /*
- * set outputOI
- */
- if (windowExpressions.size() > 0) {
- ArrayList<String> aliases = new ArrayList<String>();
- ArrayList<ObjectInspector> fieldOIs = new ArrayList<ObjectInspector>();
- for (WindowExpressionDef wEDef : windowExpressions) {
- aliases.add(wEDef.getAlias());
- fieldOIs.add(wEDef.getOI());
- }
- PTFTranslator.addInputColumnsToList(wdwOutShape, aliases, fieldOIs);
- StructObjectInspector outOI = ObjectInspectorFactory.getStandardStructObjectInspector(
- aliases, fieldOIs);
- RowResolver outRR = buildRowResolverForWindowing(wdwTFnDef, true);
- ShapeDetails outShape = setupShape(outOI, null, outRR);
- wdwTFnDef.setOutputShape(outShape);
- }
- else {
- wdwTFnDef.setOutputShape(copyShape(wdwOutShape));
}
+ PTFTranslator.addInputColumnsToList(inpShape, aliases, fieldOIs);
+ StructObjectInspector wdwOutOI = ObjectInspectorFactory.getStandardStructObjectInspector(
+ aliases, fieldOIs);
+ tFn.setWdwProcessingOutputOI(wdwOutOI);
+ RowResolver wdwOutRR = buildRowResolverForWindowing(wdwTFnDef);
+ ShapeDetails wdwOutShape = setupShape(wdwOutOI, null, wdwOutRR);
+ wdwTFnDef.setOutputShape(wdwOutShape);
tFn.setupOutputOI();
- /*
- * If we have windowExpressions then we convert to Std. Object to process;
- * we just stream these rows; no need to put in an output Partition.
- */
- if (windowExpressions.size() > 0) {
- StructObjectInspector oi = (StructObjectInspector)
- ObjectInspectorUtils.getStandardObjectInspector(wdwTFnDef.getOutputShape().getOI());
- wdwTFnDef.getOutputShape().setOI(oi);
- }
-
return ptfDesc;
}
@@ -693,7 +638,8 @@ public class PTFTranslator {
try {
serde = PTFTranslator.createLazyBinarySerDe(hCfg, OI, serdePropsMap);
- shp.setOI((StructObjectInspector) serde.getObjectInspector());
+ StructObjectInspector outOI = PTFPartition.setupPartitionOutputOI(serde, OI);
+ shp.setOI(outOI);
} catch (SerDeException se) {
throw new SemanticException(se);
}
@@ -945,23 +891,10 @@ public class PTFTranslator {
return rwsch;
}
- protected RowResolver buildRowResolverForWindowing(WindowTableFunctionDef def,
- boolean addWdwExprs) throws SemanticException {
+ protected RowResolver buildRowResolverForWindowing(WindowTableFunctionDef def)
+ throws SemanticException {
RowResolver rr = new RowResolver();
HashMap<String, WindowExpressionSpec> aliasToExprMap = windowingSpec.getAliasToWdwExpr();
- /*
- * add Window Expressions
- */
- if (addWdwExprs) {
- for (WindowExpressionDef wEDef : def.getWindowExpressions()) {
- ASTNode ast = aliasToExprMap.get(wEDef.getAlias()).getExpression();
- ColumnInfo cInfo = new ColumnInfo(wEDef.getAlias(),
- TypeInfoUtils.getTypeInfoFromObjectInspector(wEDef.getOI()),
- null,
- false);
- rr.putExpression(ast, cInfo);
- }
- }
/*
* add Window Functions
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java Thu Sep 12 01:21:10 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.Fe
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
+import org.apache.hadoop.hive.ql.exec.ListSinkOperator;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
@@ -52,6 +53,7 @@ import org.apache.hadoop.hive.ql.plan.Lo
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
/**
* Parse Context: The current parse context. This is passed to the optimizer
@@ -100,15 +102,6 @@ public class ParseContext {
*/
private LineageInfo lInfo;
- // is set to true if the expression only contains partitioning columns and not
- // any other column reference.
- // This is used to optimize select * from table where ... scenario, when the
- // where condition only references
- // partitioning columns - the partitions are identified and streamed directly
- // to the client without requiring
- // a map-reduce job
- private boolean hasNonPartCols;
-
private GlobalLimitCtx globalLimitCtx;
private HashSet<ReadEntity> semanticInputs;
@@ -117,6 +110,10 @@ public class ParseContext {
private FetchTask fetchTask;
private QueryProperties queryProperties;
+ private TableDesc fetchTabledesc;
+ private Operator<?> fetchSource;
+ private ListSinkOperator fetchSink;
+
public ParseContext() {
}
@@ -206,7 +203,6 @@ public class ParseContext {
this.destTableId = destTableId;
this.uCtx = uCtx;
this.listMapJoinOpsNoReducer = listMapJoinOpsNoReducer;
- hasNonPartCols = false;
this.groupOpToInputTables = groupOpToInputTables;
this.prunedPartitions = prunedPartitions;
this.opToSamplePruner = opToSamplePruner;
@@ -515,22 +511,6 @@ public class ParseContext {
}
/**
- * Sets the hasNonPartCols flag.
- *
- * @param val
- */
- public void setHasNonPartCols(boolean val) {
- hasNonPartCols = val;
- }
-
- /**
- * Gets the value of the hasNonPartCols flag.
- */
- public boolean getHasNonPartCols() {
- return hasNonPartCols;
- }
-
- /**
* @return the opToSamplePruner
*/
public HashMap<TableScanOperator, sampleDesc> getOpToSamplePruner() {
@@ -673,4 +653,28 @@ public class ParseContext {
public void setQueryProperties(QueryProperties queryProperties) {
this.queryProperties = queryProperties;
}
+
+ public TableDesc getFetchTabledesc() {
+ return fetchTabledesc;
+ }
+
+ public void setFetchTabledesc(TableDesc fetchTabledesc) {
+ this.fetchTabledesc = fetchTabledesc;
+ }
+
+ public Operator<?> getFetchSource() {
+ return fetchSource;
+ }
+
+ public void setFetchSource(Operator<?> fetchSource) {
+ this.fetchSource = fetchSource;
+ }
+
+ public ListSinkOperator getFetchSink() {
+ return fetchSink;
+ }
+
+ public void setFetchSink(ListSinkOperator fetchSink) {
+ this.fetchSink = fetchSink;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java Thu Sep 12 01:21:10 2013
@@ -51,22 +51,22 @@ public class QBJoinTree implements Seria
// keeps track of the right-hand-side table name of the left-semi-join, and
// its list of join keys
- private final HashMap<String, ArrayList<ASTNode>> rhsSemijoin;
+ private transient final HashMap<String, ArrayList<ASTNode>> rhsSemijoin;
// join conditions
- private ArrayList<ArrayList<ASTNode>> expressions;
+ private transient ArrayList<ArrayList<ASTNode>> expressions;
// key index to nullsafe join flag
private ArrayList<Boolean> nullsafes;
// filters
- private ArrayList<ArrayList<ASTNode>> filters;
+ private transient ArrayList<ArrayList<ASTNode>> filters;
// outerjoin-pos = other-pos:filter-len, other-pos:filter-len, ...
private int[][] filterMap;
// filters for pushing
- private ArrayList<ArrayList<ASTNode>> filtersForPushing;
+ private transient ArrayList<ArrayList<ASTNode>> filtersForPushing;
// user asked for map-side join
private boolean mapSideJoin;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/RowResolver.java Thu Sep 12 01:21:10 2013
@@ -318,4 +318,18 @@ public class RowResolver implements Seri
this.expressionMap = expressionMap;
}
+ public String[] toColumnDesc() {
+ StringBuilder cols = new StringBuilder();
+ StringBuilder colTypes = new StringBuilder();
+
+ for (ColumnInfo colInfo : getColumnInfos()) {
+ if (cols.length() > 0) {
+ cols.append(',');
+ colTypes.append(':');
+ }
+ cols.append(colInfo.getInternalName());
+ colTypes.append(colInfo.getType().getTypeName());
+ }
+ return new String[] {cols.toString(), colTypes.toString()};
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g?rev=1522098&r1=1522097&r2=1522098&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SelectClauseParser.g Thu Sep 12 01:21:10 2013
@@ -125,9 +125,9 @@ selectItem
@init { gParent.msgs.push("selection target"); }
@after { gParent.msgs.pop(); }
:
- ( selectExpression (KW_OVER ws=window_specification )?
+ ( selectExpression
((KW_AS? identifier) | (KW_AS LPAREN identifier (COMMA identifier)* RPAREN))?
- ) -> ^(TOK_SELEXPR selectExpression identifier* $ws?)
+ ) -> ^(TOK_SELEXPR selectExpression identifier*)
;
trfmClause