You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2013/07/09 11:07:43 UTC
svn commit: r1501145 [5/13] - in /hive/branches/vectorization: ./
beeline/src/java/org/apache/hive/beeline/
beeline/src/test/org/apache/hive/beeline/src/test/
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/common/metric...
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ReduceSinkDeDuplication.java Tue Jul 9 09:07:35 2013
@@ -73,6 +73,8 @@ import static org.apache.hadoop.hive.con
* If two reducer sink operators share the same partition/sort columns and order,
* they can be merged. This should happen after map join optimization because map
* join optimization will remove reduce sink operators.
+ *
+ * This optimizer removes/replaces child-RS (not parent) which is safer way for DefaultGraphWalker.
*/
public class ReduceSinkDeDuplication implements Transform{
@@ -89,9 +91,12 @@ public class ReduceSinkDeDuplication imp
// generate pruned column list for all relevant operators
ReduceSinkDeduplicateProcCtx cppCtx = new ReduceSinkDeduplicateProcCtx(pGraphContext);
+ // for auto convert map-joins, it not safe to dedup in here (todo)
boolean mergeJoins = !pctx.getConf().getBoolVar(HIVECONVERTJOIN) &&
!pctx.getConf().getBoolVar(HIVECONVERTJOINNOCONDITIONALTASK);
+ // If multiple rules can be matched with same cost, last rule will be choosen as a processor
+ // see DefaultRuleDispatcher#dispatch()
Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
opRules.put(new RuleRegExp("R1", RS + "%.*%" + RS + "%"),
ReduceSinkDeduplicateProcFactory.getReducerReducerProc());
@@ -119,8 +124,14 @@ public class ReduceSinkDeDuplication imp
class ReduceSinkDeduplicateProcCtx implements NodeProcessorCtx {
ParseContext pctx;
+
+ // For queries using script, the optimization cannot be applied without user's confirmation
+ // If script preserves alias and value for columns related to keys, user can set this true
boolean trustScript;
- // min reducer num for merged RS (to avoid query contains "order by" executed by one reducer)
+
+ // This is min number of reducer for deduped RS to avoid query executed on
+ // too small number of reducers. For example, queries GroupBy+OrderBy can be executed by
+ // only one reducer if this configuration does not prevents
int minReducer;
Set<Operator<?>> removedOps;
@@ -178,7 +189,7 @@ public class ReduceSinkDeDuplication imp
}
}
- public abstract static class AbsctractReducerReducerProc implements NodeProcessor {
+ public abstract static class AbstractReducerReducerProc implements NodeProcessor {
ReduceSinkDeduplicateProcCtx dedupCtx;
@@ -323,6 +334,8 @@ public class ReduceSinkDeDuplication imp
return result;
}
+ // for left outer joins, left alias is sorted but right alias might be not
+ // (nulls, etc.). vice versa.
private boolean isSortedTag(JoinOperator joinOp, int tag) {
for (JoinCondDesc cond : joinOp.getConf().getConds()) {
switch (cond.getType()) {
@@ -356,6 +369,10 @@ public class ReduceSinkDeDuplication imp
return -1;
}
+ /**
+ * Current RSDedup remove/replace child RS. So always copies
+ * more specific part of configurations of child RS to that of parent RS.
+ */
protected boolean merge(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer)
throws SemanticException {
int[] result = checkStatus(cRS, pRS, minReducer);
@@ -379,7 +396,15 @@ public class ReduceSinkDeDuplication imp
return true;
}
- // -1 for p to c, 1 for c to p
+ /**
+ * Returns merge directions between two RSs for criterias (ordering, number of reducers,
+ * reducer keys, partition keys). Returns null if any of categories is not mergeable.
+ *
+ * Values for each index can be -1, 0, 1
+ * 1. 0 means two configuration in the category is the same
+ * 2. for -1, configuration of parent RS is more specific than child RS
+ * 3. for 1, configuration of child RS is more specific than parent RS
+ */
private int[] checkStatus(ReduceSinkOperator cRS, ReduceSinkOperator pRS, int minReducer)
throws SemanticException {
ReduceSinkDesc cConf = cRS.getConf();
@@ -408,6 +433,11 @@ public class ReduceSinkDeDuplication imp
return new int[] {moveKeyColTo, movePartitionColTo, moveRSOrderTo, moveReducerNumTo};
}
+ /**
+ * Overlapping part of keys should be the same between parent and child.
+ * And if child has more keys than parent, non-overlapping part of keys
+ * should be backtrackable to parent.
+ */
private Integer checkExprs(List<ExprNodeDesc> ckeys, List<ExprNodeDesc> pkeys,
ReduceSinkOperator cRS, ReduceSinkOperator pRS) throws SemanticException {
Integer moveKeyColTo = 0;
@@ -419,6 +449,7 @@ public class ReduceSinkDeDuplication imp
if (pkeys == null || pkeys.isEmpty()) {
for (ExprNodeDesc ckey : ckeys) {
if (ExprNodeDescUtils.backtrack(ckey, cRS, pRS) == null) {
+ // cKey is not present in parent
return null;
}
}
@@ -430,6 +461,7 @@ public class ReduceSinkDeDuplication imp
return moveKeyColTo;
}
+ // backtrack key exprs of child to parent and compare it with parent's
protected Integer sameKeys(List<ExprNodeDesc> cexprs, List<ExprNodeDesc> pexprs,
Operator<?> child, Operator<?> parent) throws SemanticException {
int common = Math.min(cexprs.size(), pexprs.size());
@@ -438,13 +470,14 @@ public class ReduceSinkDeDuplication imp
for (; i < common; i++) {
ExprNodeDesc pexpr = pexprs.get(i);
ExprNodeDesc cexpr = ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent);
- if (!pexpr.isSame(cexpr)) {
+ if (cexpr == null || !pexpr.isSame(cexpr)) {
return null;
}
}
for (;i < limit; i++) {
if (cexprs.size() > pexprs.size()) {
if (ExprNodeDescUtils.backtrack(cexprs.get(i), child, parent) == null) {
+ // cKey is not present in parent
return null;
}
}
@@ -452,6 +485,7 @@ public class ReduceSinkDeDuplication imp
return Integer.valueOf(cexprs.size()).compareTo(pexprs.size());
}
+ // order of overlapping keys should be exactly the same
protected Integer checkOrder(String corder, String porder) {
if (corder == null || corder.trim().equals("")) {
if (porder == null || porder.trim().equals("")) {
@@ -471,6 +505,11 @@ public class ReduceSinkDeDuplication imp
return Integer.valueOf(corder.length()).compareTo(porder.length());
}
+ /**
+ * If number of reducers for RS is -1, the RS can have any number of reducers.
+ * It's generally true except for order-by or forced bucketing cases.
+ * if both of num-reducers are not -1, those number should be the same.
+ */
protected Integer checkNumReducer(int creduce, int preduce) {
if (creduce < 0) {
if (preduce < 0) {
@@ -549,6 +588,8 @@ public class ReduceSinkDeDuplication imp
return select;
}
+ // replace the cRS to SEL operator
+ // If child if cRS is EXT, EXT also should be removed
private SelectOperator replaceOperatorWithSelect(Operator<?> operator, ParseContext context)
throws SemanticException {
RowResolver inputRR = context.getOpParseCtx().get(operator).getRowResolver();
@@ -585,6 +626,8 @@ public class ReduceSinkDeDuplication imp
Operator<?> parent = getSingleParent(cRS);
if (parent instanceof GroupByOperator) {
+ // pRS-cGBYm-cRS-cGBYr (map aggregation) --> pRS-cGBYr(COMPLETE)
+ // copies desc of cGBYm to cGBYr and remove cGBYm and cRS
GroupByOperator cGBYm = (GroupByOperator) parent;
cGBYr.getConf().setKeys(cGBYm.getConf().getKeys());
@@ -597,6 +640,8 @@ public class ReduceSinkDeDuplication imp
RowResolver resolver = context.getOpParseCtx().get(cGBYm).getRowResolver();
context.getOpParseCtx().get(cGBYr).setRowResolver(resolver);
} else {
+ // pRS-cRS-cGBYr (no map aggregation) --> pRS-cGBYr(COMPLETE)
+ // revert expressions of cGBYr to that of cRS
cGBYr.getConf().setKeys(ExprNodeDescUtils.backtrack(cGBYr.getConf().getKeys(), cGBYr, cRS));
for (AggregationDesc aggr : cGBYr.getConf().getAggregators()) {
aggr.setParameters(ExprNodeDescUtils.backtrack(aggr.getParameters(), cGBYr, cRS));
@@ -655,7 +700,7 @@ public class ReduceSinkDeDuplication imp
}
}
- static class GroupbyReducerProc extends AbsctractReducerReducerProc {
+ static class GroupbyReducerProc extends AbstractReducerReducerProc {
// pRS-pGBY-cRS
public Object process(ReduceSinkOperator cRS, ParseContext context)
@@ -689,7 +734,7 @@ public class ReduceSinkDeDuplication imp
}
}
- static class JoinReducerProc extends AbsctractReducerReducerProc {
+ static class JoinReducerProc extends AbstractReducerReducerProc {
// pRS-pJOIN-cRS
public Object process(ReduceSinkOperator cRS, ParseContext context)
@@ -717,7 +762,7 @@ public class ReduceSinkDeDuplication imp
}
}
- static class ReducerReducerProc extends AbsctractReducerReducerProc {
+ static class ReducerReducerProc extends AbstractReducerReducerProc {
// pRS-cRS
public Object process(ReduceSinkOperator cRS, ParseContext context)
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcCtx.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcCtx.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcCtx.java Tue Jul 9 09:07:35 2013
@@ -28,32 +28,22 @@ import org.apache.hadoop.hive.ql.metadat
* partition pruned for the table scan and table alias.
*/
public class PcrExprProcCtx implements NodeProcessorCtx {
+ /**
+ * The table alias that is being currently processed.
+ */
+ private final String tabAlias;
+ private final List<Partition> partList;
public PcrExprProcCtx(String tabAlias, List<Partition> partList) {
- super();
this.tabAlias = tabAlias;
this.partList = partList;
}
- /**
- * The table alias that is being currently processed.
- */
- String tabAlias;
- List<Partition> partList;
-
public String getTabAlias() {
return tabAlias;
}
- public void setTabAlias(String tabAlias) {
- this.tabAlias = tabAlias;
- }
-
public List<Partition> getPartList() {
return partList;
}
-
- public void setPartList(List<Partition> partList) {
- this.partList = partList;
- }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java Tue Jul 9 09:07:35 2013
@@ -364,16 +364,14 @@ public final class PcrExprProcFactory {
Object... nodeOutputs) throws SemanticException {
ExprNodeFieldDesc fnd = (ExprNodeFieldDesc) nd;
boolean unknown = false;
- int idx = 0;
for (Object child : nodeOutputs) {
NodeInfoWrapper wrapper = (NodeInfoWrapper) child;
if (wrapper.state == WalkState.UNKNOWN) {
unknown = true;
+ break;
}
}
- assert (idx == 0);
-
if (unknown) {
return new NodeInfoWrapper(WalkState.UNKNOWN, null, fnd);
} else {
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingCtx.java Tue Jul 9 09:07:35 2013
@@ -36,6 +36,8 @@ import org.apache.hadoop.hive.ql.plan.Op
*/
public class BucketingSortingCtx implements NodeProcessorCtx {
+ boolean disableBucketing;
+
// A mapping from an operator to the columns by which it's output is bucketed
Map<Operator<? extends OperatorDesc>, List<BucketCol>> bucketedColsByOp;
// A mapping from a directory which a FileSinkOperator writes into to the columns by which that
@@ -48,7 +50,8 @@ public class BucketingSortingCtx impleme
// output is sorted
Map<String, List<SortCol>> sortedColsByDirectory;
- public BucketingSortingCtx() {
+ public BucketingSortingCtx(boolean disableBucketing) {
+ this.disableBucketing = disableBucketing;
this.bucketedColsByOp = new HashMap<Operator<? extends OperatorDesc>, List<BucketCol>>();
this.bucketedColsByDirectory = new HashMap<String, List<BucketCol>>();
this.sortedColsByOp = new HashMap<Operator<? extends OperatorDesc>, List<SortCol>>();
@@ -57,21 +60,25 @@ public class BucketingSortingCtx impleme
public List<BucketCol> getBucketedCols(Operator<? extends OperatorDesc> op) {
- return bucketedColsByOp.get(op);
+ return disableBucketing ? null : bucketedColsByOp.get(op);
}
public void setBucketedCols(Operator<? extends OperatorDesc> op, List<BucketCol> bucketCols) {
- this.bucketedColsByOp.put(op, bucketCols);
+ if (!disableBucketing) {
+ bucketedColsByOp.put(op, bucketCols);
+ }
}
public Map<String, List<BucketCol>> getBucketedColsByDirectory() {
- return bucketedColsByDirectory;
+ return disableBucketing ? null : bucketedColsByDirectory;
}
public void setBucketedColsByDirectory(Map<String, List<BucketCol>> bucketedColsByDirectory) {
- this.bucketedColsByDirectory = bucketedColsByDirectory;
+ if (!disableBucketing) {
+ this.bucketedColsByDirectory = bucketedColsByDirectory;
+ }
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/BucketingSortingInferenceOptimizer.java Tue Jul 9 09:07:35 2013
@@ -91,7 +91,9 @@ public class BucketingSortingInferenceOp
continue;
}
- BucketingSortingCtx bCtx = new BucketingSortingCtx();
+ // uses sampling, which means it's not bucketed
+ boolean disableBucketing = mapRedTask.getWork().getSamplingType() > 0;
+ BucketingSortingCtx bCtx = new BucketingSortingCtx(disableBucketing);
// RuleRegExp rules are used to match operators anywhere in the tree
// RuleExactMatch rules are used to specify exactly what the tree should look like
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java Tue Jul 9 09:07:35 2013
@@ -34,7 +34,7 @@ public class IndexWhereResolver implemen
Dispatcher dispatcher = new IndexWhereTaskDispatcher(physicalContext);
GraphWalker opGraphWalker = new DefaultGraphWalker(dispatcher);
ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(physicalContext.rootTasks);
+ topNodes.addAll(physicalContext.getRootTasks());
opGraphWalker.startWalking(topNodes, null);
return physicalContext;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MapJoinResolver.java Tue Jul 9 09:07:35 2013
@@ -73,7 +73,7 @@ public class MapJoinResolver implements
// get all the tasks nodes from root task
ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(pctx.rootTasks);
+ topNodes.addAll(pctx.getRootTasks());
// begin to walk through the task tree.
ogw.startWalking(topNodes, null);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/MetadataOnlyOptimizer.java Tue Jul 9 09:07:35 2013
@@ -171,7 +171,7 @@ public class MetadataOnlyOptimizer imple
Dispatcher disp = new MetadataOnlyTaskDispatcher(pctx);
GraphWalker ogw = new DefaultGraphWalker(disp);
ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(pctx.rootTasks);
+ topNodes.addAll(pctx.getRootTasks());
ogw.startWalking(topNodes, null);
return pctx;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalContext.java Tue Jul 9 09:07:35 2013
@@ -72,11 +72,27 @@ public class PhysicalContext {
this.context = context;
}
+ public List<Task<? extends Serializable>> getRootTasks() {
+ return rootTasks;
+ }
+
+ public void setRootTasks(List<Task<? extends Serializable>> rootTasks) {
+ this.rootTasks = rootTasks;
+ }
+
+ public Task<? extends Serializable> getFetchTask() {
+ return fetchTask;
+ }
+
+ public void setFetchTask(Task<? extends Serializable> fetchTask) {
+ this.fetchTask = fetchTask;
+ }
+
public void addToRootTask(Task<? extends Serializable> tsk){
rootTasks.add(tsk);
}
+
public void removeFromRootTask(Task<? extends Serializable> tsk){
rootTasks.remove(tsk);
}
-
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java Tue Jul 9 09:07:35 2013
@@ -67,6 +67,9 @@ public class PhysicalOptimizer {
if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVEMETADATAONLYQUERIES)) {
resolvers.add(new MetadataOnlyOptimizer());
}
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVESAMPLINGFORORDERBY)) {
+ resolvers.add(new SamplingOptimizer());
+ }
// Physical optimizers which follow this need to be careful not to invalidate the inferences
// made by this optimizer. Only optimizers which depend on the results of this one should
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SkewJoinResolver.java Tue Jul 9 09:07:35 2013
@@ -51,7 +51,7 @@ public class SkewJoinResolver implements
Dispatcher disp = new SkewJoinTaskDispatcher(pctx);
GraphWalker ogw = new DefaultGraphWalker(disp);
ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(pctx.rootTasks);
+ topNodes.addAll(pctx.getRootTasks());
ogw.startWalking(topNodes, null);
return pctx;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/SortMergeJoinTaskDispatcher.java Tue Jul 9 09:07:35 2013
@@ -118,17 +118,10 @@ public class SortMergeJoinTaskDispatcher
PartitionDesc partitionInfo = currWork.getAliasToPartnInfo().get(alias);
if (fetchWork.getTblDir() != null) {
- ArrayList<String> aliases = new ArrayList<String>();
- aliases.add(alias);
- currWork.getPathToAliases().put(fetchWork.getTblDir(), aliases);
- currWork.getPathToPartitionInfo().put(fetchWork.getTblDir(), partitionInfo);
- }
- else {
+ currWork.mergeAliasedInput(alias, fetchWork.getTblDir(), partitionInfo);
+ } else {
for (String pathDir : fetchWork.getPartDir()) {
- ArrayList<String> aliases = new ArrayList<String>();
- aliases.add(alias);
- currWork.getPathToAliases().put(pathDir, aliases);
- currWork.getPathToPartitionInfo().put(pathDir, partitionInfo);
+ currWork.mergeAliasedInput(alias, pathDir, partitionInfo);
}
}
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Tue Jul 9 09:07:35 2013
@@ -72,6 +72,7 @@ import org.apache.hadoop.mapred.TextInpu
* BaseSemanticAnalyzer.
*
*/
+@SuppressWarnings("deprecation")
public abstract class BaseSemanticAnalyzer {
protected final Hive db;
protected final HiveConf conf;
@@ -426,6 +427,8 @@ public abstract class BaseSemanticAnalyz
}
}
+ private static final int[] multiplier = new int[] {1000, 100, 10, 1};
+
@SuppressWarnings("nls")
public static String unescapeSQLString(String b) {
@@ -451,6 +454,18 @@ public abstract class BaseSemanticAnalyz
continue;
}
+ if (currentChar == '\\' && (i + 6 < b.length()) && b.charAt(i + 1) == 'u') {
+ int code = 0;
+ int base = i + 2;
+ for (int j = 0; j < 4; j++) {
+ int digit = Character.digit(b.charAt(j + base), 16);
+ code += digit * multiplier[j];
+ }
+ sb.append((char)code);
+ i += 5;
+ continue;
+ }
+
if (currentChar == '\\' && (i + 4 < b.length())) {
char i1 = b.charAt(i + 1);
char i2 = b.charAt(i + 2);
@@ -551,20 +566,22 @@ public abstract class BaseSemanticAnalyz
for (int i = 0; i < numCh; i++) {
FieldSchema col = new FieldSchema();
ASTNode child = (ASTNode) ast.getChild(i);
-
- String name = child.getChild(0).getText();
- if(lowerCase) {
- name = name.toLowerCase();
- }
- // child 0 is the name of the column
- col.setName(unescapeIdentifier(name));
- // child 1 is the type of the column
- ASTNode typeChild = (ASTNode) (child.getChild(1));
- col.setType(getTypeStringFromAST(typeChild));
-
- // child 2 is the optional comment of the column
- if (child.getChildCount() == 3) {
- col.setComment(unescapeSQLString(child.getChild(2).getText()));
+ Tree grandChild = child.getChild(0);
+ if(grandChild != null) {
+ String name = grandChild.getText();
+ if(lowerCase) {
+ name = name.toLowerCase();
+ }
+ // child 0 is the name of the column
+ col.setName(unescapeIdentifier(name));
+ // child 1 is the type of the column
+ ASTNode typeChild = (ASTNode) (child.getChild(1));
+ col.setType(getTypeStringFromAST(typeChild));
+
+ // child 2 is the optional comment of the column
+ if (child.getChildCount() == 3) {
+ col.setComment(unescapeSQLString(child.getChild(2).getText()));
+ }
}
colList.add(col);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java Tue Jul 9 09:07:35 2013
@@ -40,26 +40,31 @@ public class ExplainSemanticAnalyzer ext
super(conf);
}
+ @SuppressWarnings("unchecked")
@Override
public void analyzeInternal(ASTNode ast) throws SemanticException {
- ctx.setExplain(true);
-
- // Create a semantic analyzer for the query
- BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, (ASTNode) ast
- .getChild(0));
- sem.analyze((ASTNode) ast.getChild(0), ctx);
- sem.validate();
boolean extended = false;
boolean formatted = false;
boolean dependency = false;
+ boolean logical = false;
if (ast.getChildCount() == 2) {
int explainOptions = ast.getChild(1).getType();
formatted = (explainOptions == HiveParser.KW_FORMATTED);
extended = (explainOptions == HiveParser.KW_EXTENDED);
dependency = (explainOptions == HiveParser.KW_DEPENDENCY);
+ logical = (explainOptions == HiveParser.KW_LOGICAL);
}
+ ctx.setExplain(true);
+ ctx.setExplainLogical(logical);
+
+ // Create a semantic analyzer for the query
+ BaseSemanticAnalyzer sem = SemanticAnalyzerFactory.get(conf, (ASTNode) ast
+ .getChild(0));
+ sem.analyze((ASTNode) ast.getChild(0), ctx);
+ sem.validate();
+
ctx.setResFile(new Path(ctx.getLocalTmpFileURI()));
List<Task<? extends Serializable>> tasks = sem.getRootTasks();
Task<? extends Serializable> fetchTask = sem.getFetchTask();
@@ -72,14 +77,21 @@ public class ExplainSemanticAnalyzer ext
tasks.add(fetchTask);
}
+ ParseContext pCtx = null;
+ if (sem instanceof SemanticAnalyzer) {
+ pCtx = ((SemanticAnalyzer)sem).getParseContext();
+ }
+
Task<? extends Serializable> explTask =
TaskFactory.get(new ExplainWork(ctx.getResFile().toString(),
+ pCtx,
tasks,
((ASTNode) ast.getChild(0)).toStringTree(),
sem.getInputs(),
extended,
formatted,
- dependency),
+ dependency,
+ logical),
conf);
fieldList = explTask.getResultSchema();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/FromClauseParser.g Tue Jul 9 09:07:35 2013
@@ -125,7 +125,11 @@ lateralView
@init {gParent.msgs.push("lateral view"); }
@after {gParent.msgs.pop(); }
:
- KW_LATERAL KW_VIEW function tableAlias KW_AS identifier (COMMA identifier)* -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ KW_LATERAL KW_VIEW KW_OUTER function tableAlias KW_AS identifier (COMMA identifier)*
+ -> ^(TOK_LATERAL_VIEW_OUTER ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
+ |
+ KW_LATERAL KW_VIEW function tableAlias KW_AS identifier (COMMA identifier)*
+ -> ^(TOK_LATERAL_VIEW ^(TOK_SELECT ^(TOK_SELEXPR function identifier+ tableAlias)))
;
tableAlias
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g Tue Jul 9 09:07:35 2013
@@ -155,11 +155,13 @@ KW_RLIKE: 'RLIKE';
KW_REGEXP: 'REGEXP';
KW_TEMPORARY: 'TEMPORARY';
KW_FUNCTION: 'FUNCTION';
+KW_MACRO: 'MACRO';
KW_EXPLAIN: 'EXPLAIN';
KW_EXTENDED: 'EXTENDED';
KW_FORMATTED: 'FORMATTED';
KW_PRETTY: 'PRETTY';
KW_DEPENDENCY: 'DEPENDENCY';
+KW_LOGICAL: 'LOGICAL';
KW_SERDE: 'SERDE';
KW_WITH: 'WITH';
KW_DEFERRED: 'DEFERRED';
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g Tue Jul 9 09:07:35 2013
@@ -205,6 +205,8 @@ TOK_STRINGLITERALSEQUENCE;
TOK_CHARSETLITERAL;
TOK_CREATEFUNCTION;
TOK_DROPFUNCTION;
+TOK_CREATEMACRO;
+TOK_DROPMACRO;
TOK_CREATEVIEW;
TOK_DROPVIEW;
TOK_ALTERVIEW_AS;
@@ -238,6 +240,7 @@ TOK_RECORDREADER;
TOK_RECORDWRITER;
TOK_LEFTSEMIJOIN;
TOK_LATERAL_VIEW;
+TOK_LATERAL_VIEW_OUTER;
TOK_TABALIAS;
TOK_ANALYZE;
TOK_CREATEROLE;
@@ -555,7 +558,7 @@ statement
explainStatement
@init { msgs.push("explain statement"); }
@after { msgs.pop(); }
- : KW_EXPLAIN (explainOptions=KW_EXTENDED|explainOptions=KW_FORMATTED|explainOptions=KW_DEPENDENCY)? execStatement
+ : KW_EXPLAIN (explainOptions=KW_EXTENDED|explainOptions=KW_FORMATTED|explainOptions=KW_DEPENDENCY|explainOptions=KW_LOGICAL)? execStatement
-> ^(TOK_EXPLAIN execStatement $explainOptions?)
;
@@ -606,9 +609,11 @@ ddlStatement
| createViewStatement
| dropViewStatement
| createFunctionStatement
+ | createMacroStatement
| createIndexStatement
| dropIndexStatement
| dropFunctionStatement
+ | dropMacroStatement
| analyzeStatement
| lockStatement
| unlockStatement
@@ -1386,6 +1391,21 @@ dropFunctionStatement
-> ^(TOK_DROPFUNCTION identifier ifExists?)
;
+createMacroStatement
+@init { msgs.push("create macro statement"); }
+@after { msgs.pop(); }
+ : KW_CREATE KW_TEMPORARY KW_MACRO Identifier
+ LPAREN columnNameTypeList? RPAREN expression
+ -> ^(TOK_CREATEMACRO Identifier columnNameTypeList? expression)
+ ;
+
+dropMacroStatement
+@init { msgs.push("drop macro statement"); }
+@after { msgs.pop(); }
+ : KW_DROP KW_TEMPORARY KW_MACRO ifExists? Identifier
+ -> ^(TOK_DROPMACRO Identifier ifExists?)
+ ;
+
createViewStatement
@init {
msgs.push("create view statement");
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g Tue Jul 9 09:07:35 2013
@@ -512,5 +512,5 @@ identifier
nonReserved
:
- KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW_WHILE | KW_R
EAD | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_INNER
+ KW_TRUE | KW_FALSE | KW_LIKE | KW_EXISTS | KW_ASC | KW_DESC | KW_ORDER | KW_GROUP | KW_BY | KW_AS | KW_INSERT | KW_OVERWRITE | KW_OUTER | KW_LEFT | KW_RIGHT | KW_FULL | KW_PARTITION | KW_PARTITIONS | KW_TABLE | KW_TABLES | KW_COLUMNS | KW_INDEX | KW_INDEXES | KW_REBUILD | KW_FUNCTIONS | KW_SHOW | KW_MSCK | KW_REPAIR | KW_DIRECTORY | KW_LOCAL | KW_USING | KW_CLUSTER | KW_DISTRIBUTE | KW_SORT | KW_UNION | KW_LOAD | KW_EXPORT | KW_IMPORT | KW_DATA | KW_INPATH | KW_IS | KW_NULL | KW_CREATE | KW_EXTERNAL | KW_ALTER | KW_CHANGE | KW_FIRST | KW_AFTER | KW_DESCRIBE | KW_DROP | KW_RENAME | KW_IGNORE | KW_PROTECTION | KW_TO | KW_COMMENT | KW_BOOLEAN | KW_TINYINT | KW_SMALLINT | KW_INT | KW_BIGINT | KW_FLOAT | KW_DOUBLE | KW_DATE | KW_DATETIME | KW_TIMESTAMP | KW_DECIMAL | KW_STRING | KW_ARRAY | KW_STRUCT | KW_UNIONTYPE | KW_PARTITIONED | KW_CLUSTERED | KW_SORTED | KW_INTO | KW_BUCKETS | KW_ROW | KW_ROWS | KW_FORMAT | KW_DELIMITED | KW_FIELDS | KW_TERMINATED | KW_ESCAPED | KW_COLLECTION |
KW_ITEMS | KW_KEYS | KW_KEY_TYPE | KW_LINES | KW_STORED | KW_FILEFORMAT | KW_SEQUENCEFILE | KW_TEXTFILE | KW_RCFILE | KW_ORCFILE | KW_INPUTFORMAT | KW_OUTPUTFORMAT | KW_INPUTDRIVER | KW_OUTPUTDRIVER | KW_OFFLINE | KW_ENABLE | KW_DISABLE | KW_READONLY | KW_NO_DROP | KW_LOCATION | KW_BUCKET | KW_OUT | KW_OF | KW_PERCENT | KW_ADD | KW_REPLACE | KW_RLIKE | KW_REGEXP | KW_TEMPORARY | KW_EXPLAIN | KW_FORMATTED | KW_PRETTY | KW_DEPENDENCY | KW_LOGICAL | KW_SERDE | KW_WITH | KW_DEFERRED | KW_SERDEPROPERTIES | KW_DBPROPERTIES | KW_LIMIT | KW_SET | KW_UNSET | KW_TBLPROPERTIES | KW_IDXPROPERTIES | KW_VALUE_TYPE | KW_ELEM_TYPE | KW_MAPJOIN | KW_STREAMTABLE | KW_HOLD_DDLTIME | KW_CLUSTERSTATUS | KW_UTC | KW_UTCTIMESTAMP | KW_LONG | KW_DELETE | KW_PLUS | KW_MINUS | KW_FETCH | KW_INTERSECT | KW_VIEW | KW_IN | KW_DATABASES | KW_MATERIALIZED | KW_SCHEMA | KW_SCHEMAS | KW_GRANT | KW_REVOKE | KW_SSL | KW_UNDO | KW_LOCK | KW_LOCKS | KW_UNLOCK | KW_SHARED | KW_EXCLUSIVE | KW_PROCEDURE | KW_UNSIGNED | KW
_WHILE | KW_READ | KW_READS | KW_PURGE | KW_RANGE | KW_ANALYZE | KW_BEFORE | KW_BETWEEN | KW_BOTH | KW_BINARY | KW_CONTINUE | KW_CURSOR | KW_TRIGGER | KW_RECORDREADER | KW_RECORDWRITER | KW_SEMI | KW_LATERAL | KW_TOUCH | KW_ARCHIVE | KW_UNARCHIVE | KW_COMPUTE | KW_STATISTICS | KW_USE | KW_OPTION | KW_CONCATENATE | KW_SHOW_DATABASE | KW_UPDATE | KW_RESTRICT | KW_CASCADE | KW_SKEWED | KW_ROLLUP | KW_CUBE | KW_DIRECTORIES | KW_FOR | KW_GROUPING | KW_SETS | KW_TRUNCATE | KW_NOSCAN | KW_USER | KW_ROLE | KW_INNER
;
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java Tue Jul 9 09:07:35 2013
@@ -27,6 +27,8 @@ import java.util.List;
import java.util.Map;
import org.antlr.runtime.tree.Tree;
+import org.apache.commons.httpclient.URIException;
+import org.apache.commons.httpclient.util.URIUtil;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -82,7 +84,7 @@ public class LoadSemanticAnalyzer extend
// directory
if (!path.startsWith("/")) {
if (isLocal) {
- path = new Path(System.getProperty("user.dir"), path).toUri().toString();
+ path = URIUtil.decode( new Path(System.getProperty("user.dir"), path).toUri().toString() );
} else {
path = new Path(new Path("/user/" + System.getProperty("user.name")),
path).toString();
@@ -231,8 +233,13 @@ public class LoadSemanticAnalyzer extend
// that's just a test case.
String copyURIStr = ctx.getExternalTmpFileURI(toURI);
URI copyURI = URI.create(copyURIStr);
- rTask = TaskFactory.get(new CopyWork(fromURI.toString(), copyURIStr),
- conf);
+ try {
+ rTask = TaskFactory.get(new CopyWork(URIUtil.decode(fromURI.toString()), copyURIStr),
+ conf);
+ } catch (URIException e) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e
+ .getMessage()), e);
+ }
fromURI = copyURI;
}
@@ -261,8 +268,14 @@ public class LoadSemanticAnalyzer extend
}
- LoadTableDesc loadTableWork = new LoadTableDesc(fromURI.toString(),
- loadTmpPath, Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+ LoadTableDesc loadTableWork;
+ try {
+ loadTableWork = new LoadTableDesc(URIUtil.decode(fromURI.toString()),
+ loadTmpPath, Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
+ } catch (URIException e1) {
+ throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(fromTree, e1
+ .getMessage()), e1);
+ }
Task<? extends Serializable> childTask = TaskFactory.get(new MoveWork(getInputs(),
getOutputs(), loadTableWork, null, true), conf);
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Jul 9 09:07:35 2013
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.parse;
-import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
@@ -38,9 +37,7 @@ import org.antlr.runtime.tree.Tree;
import org.antlr.runtime.tree.TreeWizard;
import org.antlr.runtime.tree.TreeWizard.ContextVisitor;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.ObjectPair;
@@ -51,22 +48,17 @@ import org.apache.hadoop.hive.metastore.
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Order;
-import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ArchiveUtils;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.ColumnStatsTask;
-import org.apache.hadoop.hive.ql.exec.ConditionalTask;
import org.apache.hadoop.hive.ql.exec.ExecDriver;
-import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.JoinOperator;
-import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.MapRedTask;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
@@ -75,7 +67,6 @@ import org.apache.hadoop.hive.ql.exec.Re
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
-import org.apache.hadoop.hive.ql.exec.StatsTask;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -89,13 +80,9 @@ import org.apache.hadoop.hive.ql.io.Comb
import org.apache.hadoop.hive.ql.io.HiveOutputFormat;
import org.apache.hadoop.hive.ql.io.RCFileInputFormat;
import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
-import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.Rule;
-import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.metadata.DummyPartition;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -104,20 +91,7 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.optimizer.GenMRFileSink1;
-import org.apache.hadoop.hive.ql.optimizer.GenMROperator;
-import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext;
-import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
-import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink1;
-import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink2;
-import org.apache.hadoop.hive.ql.optimizer.GenMRRedSink3;
-import org.apache.hadoop.hive.ql.optimizer.GenMRTableScan1;
-import org.apache.hadoop.hive.ql.optimizer.GenMRUnion1;
-import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils;
-import org.apache.hadoop.hive.ql.optimizer.MapJoinFactory;
import org.apache.hadoop.hive.ql.optimizer.Optimizer;
-import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalContext;
-import org.apache.hadoop.hive.ql.optimizer.physical.PhysicalOptimizer;
import org.apache.hadoop.hive.ql.optimizer.unionproc.UnionProcContext;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.tableSpec.SpecType;
import org.apache.hadoop.hive.ql.parse.PTFInvocationSpec.OrderExpression;
@@ -139,8 +113,6 @@ import org.apache.hadoop.hive.ql.parse.W
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowFunctionSpec;
import org.apache.hadoop.hive.ql.parse.WindowingSpec.WindowSpec;
import org.apache.hadoop.hive.ql.plan.AggregationDesc;
-import org.apache.hadoop.hive.ql.plan.ColumnStatsDesc;
-import org.apache.hadoop.hive.ql.plan.ColumnStatsWork;
import org.apache.hadoop.hive.ql.plan.CreateTableDesc;
import org.apache.hadoop.hive.ql.plan.CreateTableLikeDesc;
import org.apache.hadoop.hive.ql.plan.CreateViewDesc;
@@ -154,7 +126,6 @@ import org.apache.hadoop.hive.ql.plan.Ex
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.ExtractDesc;
-import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc;
import org.apache.hadoop.hive.ql.plan.FilterDesc.sampleDesc;
@@ -170,8 +141,6 @@ import org.apache.hadoop.hive.ql.plan.Li
import org.apache.hadoop.hive.ql.plan.LoadFileDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
-import org.apache.hadoop.hive.ql.plan.MapredWork;
-import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc.OrderExpressionDef;
@@ -205,7 +174,6 @@ import org.apache.hadoop.hive.serde2.obj
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.mapred.InputFormat;
/**
@@ -710,7 +678,8 @@ public class SemanticAnalyzer extends Ba
"PTF invocation in a Join must have an alias"));
}
- } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
+ } else if (child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
+ child.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
// SELECT * FROM src1 LATERAL VIEW udtf() AS myTable JOIN src2 ...
// is not supported. Instead, the lateral view must be in a subquery
// SELECT * FROM (SELECT * FROM src1 LATERAL VIEW udtf() AS myTable) a
@@ -751,6 +720,7 @@ public class SemanticAnalyzer extends Ba
alias = processSubQuery(qb, next);
break;
case HiveParser.TOK_LATERAL_VIEW:
+ case HiveParser.TOK_LATERAL_VIEW_OUTER:
alias = processLateralView(qb, next);
break;
default:
@@ -849,7 +819,8 @@ public class SemanticAnalyzer extends Ba
processTable(qb, frm);
} else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
processSubQuery(qb, frm);
- } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW) {
+ } else if (frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW ||
+ frm.getToken().getType() == HiveParser.TOK_LATERAL_VIEW_OUTER) {
processLateralView(qb, frm);
} else if (isJoinToken(frm)) {
queryProperties.setHasJoin(true);
@@ -1026,6 +997,7 @@ public class SemanticAnalyzer extends Ba
skipRecursion = false;
break;
case HiveParser.TOK_LATERAL_VIEW:
+ case HiveParser.TOK_LATERAL_VIEW_OUTER:
// todo: nested LV
assert ast.getChildCount() == 1;
qb.getParseInfo().getDestToLateralView().put(ctx_1.dest, ast);
@@ -1428,6 +1400,10 @@ public class SemanticAnalyzer extends Ba
break;
case HiveParser.Number:
case HiveParser.StringLiteral:
+ case HiveParser.BigintLiteral:
+ case HiveParser.SmallintLiteral:
+ case HiveParser.TinyintLiteral:
+ case HiveParser.DecimalLiteral:
case HiveParser.TOK_STRINGLITERALSEQUENCE:
case HiveParser.TOK_CHARSETLITERAL:
case HiveParser.KW_TRUE:
@@ -2441,7 +2417,7 @@ public class SemanticAnalyzer extends Ba
throws SemanticException {
ASTNode selExprList = qb.getParseInfo().getSelForClause(dest);
- Operator<?> op = genSelectPlan(selExprList, qb, input);
+ Operator<?> op = genSelectPlan(selExprList, qb, input, false);
if (LOG.isDebugEnabled()) {
LOG.debug("Created Select Plan for clause: " + dest);
@@ -2452,7 +2428,7 @@ public class SemanticAnalyzer extends Ba
@SuppressWarnings("nls")
private Operator<?> genSelectPlan(ASTNode selExprList, QB qb,
- Operator<?> input) throws SemanticException {
+ Operator<?> input, boolean outerLV) throws SemanticException {
if (LOG.isDebugEnabled()) {
LOG.debug("tree: " + selExprList.toStringTree());
@@ -2691,7 +2667,7 @@ public class SemanticAnalyzer extends Ba
if (isUDTF) {
output = genUDTFPlan(genericUDTF, udtfTableAlias, udtfColAliases, qb,
- output);
+ output, outerLV);
}
if (LOG.isDebugEnabled()) {
LOG.debug("Created Select Plan row schema: " + out_rwsch.toString());
@@ -2898,11 +2874,11 @@ public class SemanticAnalyzer extends Ba
*/
@SuppressWarnings("nls")
private Operator genGroupByPlanGroupByOperator(QBParseInfo parseInfo,
- String dest, Operator reduceSinkOperatorInfo, GroupByDesc.Mode mode,
+ String dest, Operator input, ReduceSinkOperator rs, GroupByDesc.Mode mode,
Map<String, GenericUDAFEvaluator> genericUDAFEvaluators)
throws SemanticException {
RowResolver groupByInputRowResolver = opParseCtx
- .get(reduceSinkOperatorInfo).getRowResolver();
+ .get(input).getRowResolver();
RowResolver groupByOutputRowResolver = new RowResolver();
groupByOutputRowResolver.setIsExprResolver(true);
ArrayList<ExprNodeDesc> groupByKeys = new ArrayList<ExprNodeDesc>();
@@ -2933,15 +2909,11 @@ public class SemanticAnalyzer extends Ba
// get the last colName for the reduce KEY
// it represents the column name corresponding to distinct aggr, if any
String lastKeyColName = null;
- List<ExprNodeDesc> reduceValues = null;
- if (reduceSinkOperatorInfo.getConf() instanceof ReduceSinkDesc) {
- List<String> inputKeyCols = ((ReduceSinkDesc)
- reduceSinkOperatorInfo.getConf()).getOutputKeyColumnNames();
- if (inputKeyCols.size() > 0) {
- lastKeyColName = inputKeyCols.get(inputKeyCols.size() - 1);
- }
- reduceValues = ((ReduceSinkDesc) reduceSinkOperatorInfo.getConf()).getValueCols();
+ List<String> inputKeyCols = ((ReduceSinkDesc) rs.getConf()).getOutputKeyColumnNames();
+ if (inputKeyCols.size() > 0) {
+ lastKeyColName = inputKeyCols.get(inputKeyCols.size() - 1);
}
+ List<ExprNodeDesc> reduceValues = ((ReduceSinkDesc) rs.getConf()).getValueCols();
int numDistinctUDFs = 0;
for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
ASTNode value = entry.getValue();
@@ -3018,7 +2990,7 @@ public class SemanticAnalyzer extends Ba
new GroupByDesc(mode, outputColumnNames, groupByKeys, aggregations,
false, groupByMemoryUsage, memoryThreshold, null, false, 0, numDistinctUDFs > 0),
new RowSchema(groupByOutputRowResolver.getColumnInfos()),
- reduceSinkOperatorInfo), groupByOutputRowResolver);
+ input), groupByOutputRowResolver);
op.setColumnExprMap(colExprMap);
return op;
}
@@ -3486,7 +3458,7 @@ public class SemanticAnalyzer extends Ba
* @throws SemanticException
*/
@SuppressWarnings("nls")
- private Operator genGroupByPlanReduceSinkOperator(QB qb,
+ private ReduceSinkOperator genGroupByPlanReduceSinkOperator(QB qb,
String dest,
Operator inputOperatorInfo,
List<ASTNode> grpByExprs,
@@ -3676,7 +3648,7 @@ public class SemanticAnalyzer extends Ba
}
@SuppressWarnings("nls")
- private Operator genCommonGroupByPlanReduceSinkOperator(QB qb, List<String> dests,
+ private ReduceSinkOperator genCommonGroupByPlanReduceSinkOperator(QB qb, List<String> dests,
Operator inputOperatorInfo) throws SemanticException {
RowResolver reduceSinkInputRowResolver = opParseCtx.get(inputOperatorInfo)
@@ -4023,7 +3995,7 @@ public class SemanticAnalyzer extends Ba
}
// ////// 1. Generate ReduceSinkOperator
- Operator reduceSinkOperatorInfo =
+ ReduceSinkOperator reduceSinkOperatorInfo =
genGroupByPlanReduceSinkOperator(qb,
dest,
input,
@@ -4036,7 +4008,7 @@ public class SemanticAnalyzer extends Ba
// ////// 2. Generate GroupbyOperator
Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo,
- dest, reduceSinkOperatorInfo, GroupByDesc.Mode.COMPLETE, null);
+ dest, reduceSinkOperatorInfo, reduceSinkOperatorInfo, GroupByDesc.Mode.COMPLETE, null);
return groupByOperatorInfo;
}
@@ -4106,7 +4078,8 @@ public class SemanticAnalyzer extends Ba
Operator select = insertSelectAllPlanForGroupBy(selectInput);
// Generate ReduceSinkOperator
- Operator reduceSinkOperatorInfo = genCommonGroupByPlanReduceSinkOperator(qb, dests, select);
+ ReduceSinkOperator reduceSinkOperatorInfo =
+ genCommonGroupByPlanReduceSinkOperator(qb, dests, select);
// It is assumed throughout the code that a reducer has a single child, add a
// ForwardOperator so that we can add multiple filter/group by operators as children
@@ -4126,7 +4099,7 @@ public class SemanticAnalyzer extends Ba
// Generate GroupbyOperator
Operator groupByOperatorInfo = genGroupByPlanGroupByOperator(parseInfo,
- dest, curr, GroupByDesc.Mode.COMPLETE, null);
+ dest, curr, reduceSinkOperatorInfo, GroupByDesc.Mode.COMPLETE, null);
curr = genPostGroupByBodyPlan(groupByOperatorInfo, dest, qb);
}
@@ -4268,7 +4241,7 @@ public class SemanticAnalyzer extends Ba
// DISTINCT
// operator. We set the numPartitionColumns to -1 for this purpose. This is
// captured by WritableComparableHiveObject.hashCode() function.
- Operator reduceSinkOperatorInfo =
+ ReduceSinkOperator reduceSinkOperatorInfo =
genGroupByPlanReduceSinkOperator(qb,
dest,
input,
@@ -4283,7 +4256,7 @@ public class SemanticAnalyzer extends Ba
Map<String, GenericUDAFEvaluator> genericUDAFEvaluators =
new LinkedHashMap<String, GenericUDAFEvaluator>();
GroupByOperator groupByOperatorInfo = (GroupByOperator) genGroupByPlanGroupByOperator(
- parseInfo, dest, reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIAL1,
+ parseInfo, dest, reduceSinkOperatorInfo, reduceSinkOperatorInfo, GroupByDesc.Mode.PARTIAL1,
genericUDAFEvaluators);
int numReducers = -1;
@@ -5474,7 +5447,7 @@ public class SemanticAnalyzer extends Ba
private Operator genUDTFPlan(GenericUDTF genericUDTF,
String outputTableAlias, ArrayList<String> colAliases, QB qb,
- Operator input) throws SemanticException {
+ Operator input, boolean outerLV) throws SemanticException {
// No GROUP BY / DISTRIBUTE BY / SORT BY / CLUSTER BY
QBParseInfo qbp = qb.getParseInfo();
@@ -5552,7 +5525,7 @@ public class SemanticAnalyzer extends Ba
// Add the UDTFOperator to the operator DAG
Operator<?> udtf = putOpInsertMap(OperatorFactory.getAndMakeChild(
- new UDTFDesc(genericUDTF), new RowSchema(out_rwsch.getColumnInfos()),
+ new UDTFDesc(genericUDTF, outerLV), new RowSchema(out_rwsch.getColumnInfos()),
input), out_rwsch);
return udtf;
}
@@ -8148,7 +8121,8 @@ public class SemanticAnalyzer extends Ba
// Get the UDTF Path
QB blankQb = new QB(null, null, false);
Operator udtfPath = genSelectPlan((ASTNode) lateralViewTree
- .getChild(0), blankQb, lvForward);
+ .getChild(0), blankQb, lvForward,
+ lateralViewTree.getType() == HiveParser.TOK_LATERAL_VIEW_OUTER);
// add udtf aliases to QB
for (String udtfAlias : blankQb.getAliases()) {
qb.addAlias(udtfAlias);
@@ -8218,434 +8192,6 @@ public class SemanticAnalyzer extends Ba
}
}
- /**
- * A helper function to generate a column stats task on top of map-red task. The column stats
- * task fetches from the output of the map-red task, constructs the column stats object and
- * persists it to the metastore.
- *
- * This method generates a plan with a column stats task on top of map-red task and sets up the
- * appropriate metadata to be used during execution.
- *
- * @param qb
- */
- private void genColumnStatsTask(QB qb) {
- QBParseInfo qbParseInfo = qb.getParseInfo();
- ColumnStatsTask cStatsTask = null;
- ColumnStatsWork cStatsWork = null;
- FetchWork fetch = null;
- String tableName = qbParseInfo.getTableName();
- String partName = qbParseInfo.getPartName();
- List<String> colName = qbParseInfo.getColName();
- List<String> colType = qbParseInfo.getColType();
- boolean isTblLevel = qbParseInfo.isTblLvl();
-
- String cols = loadFileWork.get(0).getColumns();
- String colTypes = loadFileWork.get(0).getColumnTypes();
-
- String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
- TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
-
- fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(),
- resultTab, qb.getParseInfo().getOuterQueryLimit());
-
- ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName, partName,
- colName, colType, isTblLevel);
- cStatsWork = new ColumnStatsWork(fetch, cStatsDesc);
- cStatsTask = (ColumnStatsTask) TaskFactory.get(cStatsWork, conf);
- rootTasks.add(cStatsTask);
- }
-
- @SuppressWarnings("nls")
- private void genMapRedTasks(ParseContext pCtx) throws SemanticException {
- boolean isCStats = qb.isAnalyzeRewrite();
-
- if (pCtx.getFetchTask() != null) {
- // replaced by single fetch task
- initParseCtx(pCtx);
- return;
- }
-
- initParseCtx(pCtx);
- List<Task<MoveWork>> mvTask = new ArrayList<Task<MoveWork>>();
-
- /*
- * In case of a select, use a fetch task instead of a move task.
- * If the select is from analyze table column rewrite, don't create a fetch task. Instead create
- * a column stats task later.
- */
- if (qb.getIsQuery() && !isCStats) {
- if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
- throw new SemanticException(ErrorMsg.GENERIC_ERROR.getMsg());
- }
- String cols = loadFileWork.get(0).getColumns();
- String colTypes = loadFileWork.get(0).getColumnTypes();
-
- String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
- TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);
-
- FetchWork fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(),
- resultTab, qb.getParseInfo().getOuterQueryLimit());
-
- FetchTask fetchTask = (FetchTask) TaskFactory.get(fetch, conf);
- setFetchTask(fetchTask);
-
- // For the FetchTask, the limit optimiztion requires we fetch all the rows
- // in memory and count how many rows we get. It's not practical if the
- // limit factor is too big
- int fetchLimit = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVELIMITOPTMAXFETCH);
- if (globalLimitCtx.isEnable() && globalLimitCtx.getGlobalLimit() > fetchLimit) {
- LOG.info("For FetchTask, LIMIT " + globalLimitCtx.getGlobalLimit() + " > " + fetchLimit
- + ". Doesn't qualify limit optimiztion.");
- globalLimitCtx.disableOpt();
- }
- } else if (!isCStats) {
- for (LoadTableDesc ltd : loadTableWork) {
- Task<MoveWork> tsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
- conf);
- mvTask.add(tsk);
- // Check to see if we are stale'ing any indexes and auto-update them if we want
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEINDEXAUTOUPDATE)) {
- IndexUpdater indexUpdater = new IndexUpdater(loadTableWork, getInputs(), conf);
- try {
- List<Task<? extends Serializable>> indexUpdateTasks = indexUpdater
- .generateUpdateTasks();
- for (Task<? extends Serializable> updateTask : indexUpdateTasks) {
- tsk.addDependentTask(updateTask);
- }
- } catch (HiveException e) {
- console
- .printInfo("WARNING: could not auto-update stale indexes, which are not in sync");
- }
- }
- }
-
- boolean oneLoadFile = true;
- for (LoadFileDesc lfd : loadFileWork) {
- if (qb.isCTAS()) {
- assert (oneLoadFile); // should not have more than 1 load file for
- // CTAS
- // make the movetask's destination directory the table's destination.
- String location = qb.getTableDesc().getLocation();
- if (location == null) {
- // get the table's default location
- Table dumpTable;
- Path targetPath;
- try {
- dumpTable = db.newTable(qb.getTableDesc().getTableName());
- if (!db.databaseExists(dumpTable.getDbName())) {
- throw new SemanticException("ERROR: The database " + dumpTable.getDbName()
- + " does not exist.");
- }
- Warehouse wh = new Warehouse(conf);
- targetPath = wh.getTablePath(db.getDatabase(dumpTable.getDbName()), dumpTable
- .getTableName());
- } catch (HiveException e) {
- throw new SemanticException(e);
- } catch (MetaException e) {
- throw new SemanticException(e);
- }
-
- location = targetPath.toString();
- }
- lfd.setTargetDir(location);
-
- oneLoadFile = false;
- }
- mvTask.add(TaskFactory.get(new MoveWork(null, null, null, lfd, false),
- conf));
- }
- }
-
- // generate map reduce plans
- ParseContext tempParseContext = getParseContext();
- GenMRProcContext procCtx = new GenMRProcContext(
- conf,
- new HashMap<Operator<? extends OperatorDesc>, Task<? extends Serializable>>(),
- new ArrayList<Operator<? extends OperatorDesc>>(), tempParseContext,
- mvTask, rootTasks,
- new LinkedHashMap<Operator<? extends OperatorDesc>, GenMapRedCtx>(),
- inputs, outputs);
-
- // create a walker which walks the tree in a DFS manner while maintaining
- // the operator stack.
- // The dispatcher generates the plan from the operator tree
- Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
- opRules.put(new RuleRegExp(new String("R1"),
- TableScanOperator.getOperatorName() + "%"),
- new GenMRTableScan1());
- opRules.put(new RuleRegExp(new String("R2"),
- TableScanOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"),
- new GenMRRedSink1());
- opRules.put(new RuleRegExp(new String("R3"),
- ReduceSinkOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"),
- new GenMRRedSink2());
- opRules.put(new RuleRegExp(new String("R4"),
- FileSinkOperator.getOperatorName() + "%"),
- new GenMRFileSink1());
- opRules.put(new RuleRegExp(new String("R5"),
- UnionOperator.getOperatorName() + "%"),
- new GenMRUnion1());
- opRules.put(new RuleRegExp(new String("R6"),
- UnionOperator.getOperatorName() + "%.*" + ReduceSinkOperator.getOperatorName() + "%"),
- new GenMRRedSink3());
- opRules.put(new RuleRegExp(new String("R7"),
- MapJoinOperator.getOperatorName() + "%"),
- MapJoinFactory.getTableScanMapJoin());
-
- // The dispatcher fires the processor corresponding to the closest matching
- // rule and passes the context along
- Dispatcher disp = new DefaultRuleDispatcher(new GenMROperator(), opRules,
- procCtx);
-
- GraphWalker ogw = new GenMapRedWalker(disp);
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.addAll(topOps.values());
- ogw.startWalking(topNodes, null);
-
- /*
- * If the query was the result of analyze table column compute statistics rewrite, create
- * a column stats task instead of a fetch task to persist stats to the metastore.
- */
- if (isCStats) {
- genColumnStatsTask(qb);
- }
-
- // reduce sink does not have any kids - since the plan by now has been
- // broken up into multiple
- // tasks, iterate over all tasks.
- // For each task, go over all operators recursively
- for (Task<? extends Serializable> rootTask : rootTasks) {
- breakTaskTree(rootTask);
- }
-
- // For each task, set the key descriptor for the reducer
- for (Task<? extends Serializable> rootTask : rootTasks) {
- GenMapRedUtils.setKeyAndValueDescForTaskTree(rootTask);
- }
-
- // If a task contains an operator which instructs bucketizedhiveinputformat
- // to be used, please do so
- for (Task<? extends Serializable> rootTask : rootTasks) {
- setInputFormat(rootTask);
- }
-
- PhysicalContext physicalContext = new PhysicalContext(conf,
- getParseContext(), ctx, rootTasks, fetchTask);
- PhysicalOptimizer physicalOptimizer = new PhysicalOptimizer(
- physicalContext, conf);
- physicalOptimizer.optimize();
-
- // For each operator, generate the counters if needed
- if (HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVEJOBPROGRESS)) {
- for (Task<? extends Serializable> rootTask : rootTasks) {
- generateCountersTask(rootTask);
- }
- }
-
- decideExecMode(rootTasks, ctx, globalLimitCtx);
-
- if (qb.isCTAS()) {
- // generate a DDL task and make it a dependent task of the leaf
- CreateTableDesc crtTblDesc = qb.getTableDesc();
-
- crtTblDesc.validate();
-
- // Clear the output for CTAS since we don't need the output from the
- // mapredWork, the
- // DDLWork at the tail of the chain will have the output
- getOutputs().clear();
-
- Task<? extends Serializable> crtTblTask = TaskFactory.get(new DDLWork(
- getInputs(), getOutputs(), crtTblDesc), conf);
-
- // find all leaf tasks and make the DDLTask as a dependent task of all of
- // them
- HashSet<Task<? extends Serializable>> leaves = new HashSet<Task<? extends Serializable>>();
- getLeafTasks(rootTasks, leaves);
- assert (leaves.size() > 0);
- for (Task<? extends Serializable> task : leaves) {
- if (task instanceof StatsTask) {
- // StatsTask require table to already exist
- for (Task<? extends Serializable> parentOfStatsTask : task.getParentTasks()) {
- parentOfStatsTask.addDependentTask(crtTblTask);
- }
- for (Task<? extends Serializable> parentOfCrtTblTask : crtTblTask.getParentTasks()) {
- parentOfCrtTblTask.removeDependentTask(task);
- }
- crtTblTask.addDependentTask(task);
- } else {
- task.addDependentTask(crtTblTask);
- }
- }
- }
-
- if (globalLimitCtx.isEnable() && fetchTask != null) {
- int fetchLimit = HiveConf.getIntVar(conf, HiveConf.ConfVars.HIVELIMITOPTMAXFETCH);
- LOG.info("set least row check for FetchTask: " + globalLimitCtx.getGlobalLimit());
- fetchTask.getWork().setLeastNumRows(globalLimitCtx.getGlobalLimit());
- }
-
- if (globalLimitCtx.isEnable() && globalLimitCtx.getLastReduceLimitDesc() != null) {
- LOG.info("set least row check for LimitDesc: " + globalLimitCtx.getGlobalLimit());
- globalLimitCtx.getLastReduceLimitDesc().setLeastRows(globalLimitCtx.getGlobalLimit());
- List<ExecDriver> mrTasks = Utilities.getMRTasks(rootTasks);
- for (ExecDriver tsk : mrTasks) {
- tsk.setRetryCmdWhenFail(true);
- }
- }
- }
-
- /**
- * Find all leaf tasks of the list of root tasks.
- */
- private void getLeafTasks(List<Task<? extends Serializable>> rootTasks,
- HashSet<Task<? extends Serializable>> leaves) {
-
- for (Task<? extends Serializable> root : rootTasks) {
- getLeafTasks(root, leaves);
- }
- }
-
- private void getLeafTasks(Task<? extends Serializable> task,
- HashSet<Task<? extends Serializable>> leaves) {
- if (task.getDependentTasks() == null) {
- if (!leaves.contains(task)) {
- leaves.add(task);
- }
- } else {
- getLeafTasks(task.getDependentTasks(), leaves);
- }
- }
-
- // loop over all the tasks recursviely
- private void generateCountersTask(Task<? extends Serializable> task) {
- if (task instanceof ExecDriver) {
- HashMap<String, Operator<? extends OperatorDesc>> opMap = ((MapredWork) task
- .getWork()).getAliasToWork();
- if (!opMap.isEmpty()) {
- for (Operator<? extends OperatorDesc> op : opMap.values()) {
- generateCountersOperator(op);
- }
- }
-
- Operator<? extends OperatorDesc> reducer = ((MapredWork) task.getWork())
- .getReducer();
- if (reducer != null) {
- LOG.info("Generating counters for operator " + reducer);
- generateCountersOperator(reducer);
- }
- } else if (task instanceof ConditionalTask) {
- List<Task<? extends Serializable>> listTasks = ((ConditionalTask) task)
- .getListTasks();
- for (Task<? extends Serializable> tsk : listTasks) {
- generateCountersTask(tsk);
- }
- }
-
- // Start the counters from scratch - a hack for hadoop 17.
- Operator.resetLastEnumUsed();
-
- if (task.getChildTasks() == null) {
- return;
- }
-
- for (Task<? extends Serializable> childTask : task.getChildTasks()) {
- generateCountersTask(childTask);
- }
- }
-
- private void generateCountersOperator(Operator<? extends OperatorDesc> op) {
- op.assignCounterNameToEnum();
-
- if (op.getChildOperators() == null) {
- return;
- }
-
- for (Operator<? extends OperatorDesc> child : op.getChildOperators()) {
- generateCountersOperator(child);
- }
- }
-
- // loop over all the tasks recursviely
- private void breakTaskTree(Task<? extends Serializable> task) {
-
- if (task instanceof ExecDriver) {
- HashMap<String, Operator<? extends OperatorDesc>> opMap = ((MapredWork) task
- .getWork()).getAliasToWork();
- if (!opMap.isEmpty()) {
- for (Operator<? extends OperatorDesc> op : opMap.values()) {
- breakOperatorTree(op);
- }
- }
- } else if (task instanceof ConditionalTask) {
- List<Task<? extends Serializable>> listTasks = ((ConditionalTask) task)
- .getListTasks();
- for (Task<? extends Serializable> tsk : listTasks) {
- breakTaskTree(tsk);
- }
- }
-
- if (task.getChildTasks() == null) {
- return;
- }
-
- for (Task<? extends Serializable> childTask : task.getChildTasks()) {
- breakTaskTree(childTask);
- }
- }
-
- // loop over all the operators recursviely
- private void breakOperatorTree(Operator<? extends OperatorDesc> topOp) {
- if (topOp instanceof ReduceSinkOperator) {
- topOp.setChildOperators(null);
- }
-
- if (topOp.getChildOperators() == null) {
- return;
- }
-
- for (Operator<? extends OperatorDesc> op : topOp.getChildOperators()) {
- breakOperatorTree(op);
- }
- }
-
- private void setInputFormat(MapredWork work, Operator<? extends OperatorDesc> op) {
- if (op.isUseBucketizedHiveInputFormat()) {
- work.setUseBucketizedHiveInputFormat(true);
- return;
- }
-
- if (op.getChildOperators() != null) {
- for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) {
- setInputFormat(work, childOp);
- }
- }
- }
-
- // loop over all the tasks recursviely
- private void setInputFormat(Task<? extends Serializable> task) {
- if (task instanceof ExecDriver) {
- MapredWork work = (MapredWork) task.getWork();
- HashMap<String, Operator<? extends OperatorDesc>> opMap = work.getAliasToWork();
- if (!opMap.isEmpty()) {
- for (Operator<? extends OperatorDesc> op : opMap.values()) {
- setInputFormat(work, op);
- }
- }
- } else if (task instanceof ConditionalTask) {
- List<Task<? extends Serializable>> listTasks = ((ConditionalTask) task).getListTasks();
- for (Task<? extends Serializable> tsk : listTasks) {
- setInputFormat(tsk);
- }
- }
-
- if (task.getChildTasks() != null) {
- for (Task<? extends Serializable> childTask : task.getChildTasks()) {
- setInputFormat(childTask);
- }
- }
- }
-
@SuppressWarnings("nls")
public Phase1Ctx initPhase1Ctx() {
@@ -8773,9 +8319,14 @@ public class SemanticAnalyzer extends Ba
setColumnAccessInfo(columnAccessAnalyzer.analyzeColumnAccess());
}
- // At this point we have the complete operator tree
- // from which we want to find the reduce operator
- genMapRedTasks(pCtx);
+ if (!ctx.getExplainLogical()) {
+ // At this point we have the complete operator tree
+ // from which we want to create the map-reduce plan
+ MapReduceCompiler compiler = new MapReduceCompiler();
+ compiler.init(conf, console, db);
+ compiler.compile(pCtx, rootTasks, inputs, outputs);
+ fetchTask = pCtx.getFetchTask();
+ }
LOG.info("Completed plan generation");
@@ -9563,107 +9114,6 @@ public class SemanticAnalyzer extends Ba
}
}
- private void decideExecMode(List<Task<? extends Serializable>> rootTasks, Context ctx,
- GlobalLimitCtx globalLimitCtx)
- throws SemanticException {
-
- // bypass for explain queries for now
- if (ctx.getExplain()) {
- return;
- }
-
- // user has told us to run in local mode or doesn't want auto-local mode
- if (ctx.isLocalOnlyExecutionMode() ||
- !conf.getBoolVar(HiveConf.ConfVars.LOCALMODEAUTO)) {
- return;
- }
-
- final Context lCtx = ctx;
- PathFilter p = new PathFilter() {
- public boolean accept(Path file) {
- return !lCtx.isMRTmpFileURI(file.toUri().getPath());
- }
- };
- List<ExecDriver> mrtasks = Utilities.getMRTasks(rootTasks);
-
- // map-reduce jobs will be run locally based on data size
- // first find out if any of the jobs needs to run non-locally
- boolean hasNonLocalJob = false;
- for (ExecDriver mrtask : mrtasks) {
- try {
- ContentSummary inputSummary = Utilities.getInputSummary
- (ctx, (MapredWork) mrtask.getWork(), p);
- int numReducers = getNumberOfReducers(mrtask.getWork(), conf);
-
- long estimatedInput;
-
- if (globalLimitCtx != null && globalLimitCtx.isEnable()) {
- // If the global limit optimization is triggered, we will
- // estimate input data actually needed based on limit rows.
- // estimated Input = (num_limit * max_size_per_row) * (estimated_map + 2)
- //
- long sizePerRow = HiveConf.getLongVar(conf,
- HiveConf.ConfVars.HIVELIMITMAXROWSIZE);
- estimatedInput = globalLimitCtx.getGlobalLimit() * sizePerRow;
- long minSplitSize = HiveConf.getLongVar(conf,
- HiveConf.ConfVars.MAPREDMINSPLITSIZE);
- long estimatedNumMap = inputSummary.getLength() / minSplitSize + 1;
- estimatedInput = estimatedInput * (estimatedNumMap + 1);
- } else {
- estimatedInput = inputSummary.getLength();
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Task: " + mrtask.getId() + ", Summary: " +
- inputSummary.getLength() + "," + inputSummary.getFileCount() + ","
- + numReducers + ", estimated Input: " + estimatedInput);
- }
-
- if (MapRedTask.isEligibleForLocalMode(conf, numReducers,
- estimatedInput, inputSummary.getFileCount()) != null) {
- hasNonLocalJob = true;
- break;
- } else {
- mrtask.setLocalMode(true);
- }
- } catch (IOException e) {
- throw new SemanticException(e);
- }
- }
-
- if (!hasNonLocalJob) {
- // Entire query can be run locally.
- // Save the current tracker value and restore it when done.
- ctx.setOriginalTracker(ShimLoader.getHadoopShims().getJobLauncherRpcAddress(conf));
- ShimLoader.getHadoopShims().setJobLauncherRpcAddress(conf, "local");
- console.printInfo("Automatically selecting local only mode for query");
-
- // If all the tasks can be run locally, we can use local disk for
- // storing intermediate data.
-
- /**
- * This code is commented out pending further testing/development
- * for (Task<? extends OperatorDesc> t: rootTasks)
- * t.localizeMRTmpFiles(ctx);
- */
- }
- }
-
- /**
- * Make a best guess at trying to find the number of reducers
- */
- private static int getNumberOfReducers(MapredWork mrwork, HiveConf conf) {
- if (mrwork.getReducer() == null) {
- return 0;
- }
-
- if (mrwork.getNumReduceTasks() >= 0) {
- return mrwork.getNumReduceTasks();
- }
-
- return conf.getIntVar(HiveConf.ConfVars.HADOOPNUMREDUCERS);
- }
-
// Process the position alias in GROUPBY and ORDERBY
private void processPositionAlias(ASTNode ast) throws SemanticException {
if (HiveConf.getBoolVar(conf,
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java Tue Jul 9 09:07:35 2013
@@ -70,6 +70,8 @@ public final class SemanticAnalyzerFacto
commandType.put(HiveParser.TOK_SHOWLOCKS, HiveOperation.SHOWLOCKS);
commandType.put(HiveParser.TOK_CREATEFUNCTION, HiveOperation.CREATEFUNCTION);
commandType.put(HiveParser.TOK_DROPFUNCTION, HiveOperation.DROPFUNCTION);
+ commandType.put(HiveParser.TOK_CREATEMACRO, HiveOperation.CREATEMACRO);
+ commandType.put(HiveParser.TOK_DROPMACRO, HiveOperation.DROPMACRO);
commandType.put(HiveParser.TOK_CREATEVIEW, HiveOperation.CREATEVIEW);
commandType.put(HiveParser.TOK_DROPVIEW, HiveOperation.DROPVIEW);
commandType.put(HiveParser.TOK_CREATEINDEX, HiveOperation.CREATEINDEX);
@@ -223,6 +225,9 @@ public final class SemanticAnalyzerFacto
case HiveParser.TOK_ANALYZE:
return new ColumnStatsSemanticAnalyzer(conf, tree);
+ case HiveParser.TOK_CREATEMACRO:
+ case HiveParser.TOK_DROPMACRO:
+ return new MacroSemanticAnalyzer(conf);
default:
return new SemanticAnalyzer(conf);
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverSkewJoin.java Tue Jul 9 09:07:35 2013
@@ -31,6 +31,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Task;
+import org.apache.hadoop.hive.ql.exec.Utilities;
/**
* ConditionalResolverSkewJoin.
@@ -93,7 +94,7 @@ public class ConditionalResolverSkewJoin
String path = entry.getKey();
Path dirPath = new Path(path);
FileSystem inpFs = dirPath.getFileSystem(conf);
- FileStatus[] fstatus = inpFs.listStatus(dirPath);
+ FileStatus[] fstatus = Utilities.listStatusIfExists(dirPath, inpFs);
if (fstatus != null && fstatus.length > 0) {
Task <? extends Serializable> task = entry.getValue();
List<Task <? extends Serializable>> parentOps = task.getParentTasks();
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java Tue Jul 9 09:07:35 2013
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.parse.ParseContext;
/**
* ExplainWork.
@@ -37,20 +38,26 @@ public class ExplainWork implements Seri
private ArrayList<Task<? extends Serializable>> rootTasks;
private String astStringTree;
private HashSet<ReadEntity> inputs;
+ private ParseContext pCtx;
+
boolean extended;
boolean formatted;
boolean dependency;
+ boolean logical;
+
public ExplainWork() {
}
public ExplainWork(String resFile,
+ ParseContext pCtx,
List<Task<? extends Serializable>> rootTasks,
String astStringTree,
HashSet<ReadEntity> inputs,
boolean extended,
boolean formatted,
- boolean dependency) {
+ boolean dependency,
+ boolean logical) {
this.resFile = resFile;
this.rootTasks = new ArrayList<Task<? extends Serializable>>(rootTasks);
this.astStringTree = astStringTree;
@@ -58,6 +65,8 @@ public class ExplainWork implements Seri
this.extended = extended;
this.formatted = formatted;
this.dependency = dependency;
+ this.logical = logical;
+ this.pCtx = pCtx;
}
public String getResFile() {
@@ -115,4 +124,21 @@ public class ExplainWork implements Seri
public void setFormatted(boolean formatted) {
this.formatted = formatted;
}
+
+ public ParseContext getParseContext() {
+ return pCtx;
+ }
+
+ public void setParseContext(ParseContext pCtx) {
+ this.pCtx = pCtx;
+ }
+
+ public boolean isLogical() {
+ return logical;
+ }
+
+ public void setLogical(boolean logical) {
+ this.logical = logical;
+ }
+
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java Tue Jul 9 09:07:35 2013
@@ -103,7 +103,11 @@ public class ExprNodeConstantDesc extend
if (!typeInfo.equals(dest.getTypeInfo())) {
return false;
}
- if (!value.equals(dest.getValue())) {
+ if (value == null) {
+ if (dest.getValue() != null) {
+ return false;
+ }
+ } else if (!value.equals(dest.getValue())) {
return false;
}
Modified: hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java
URL: http://svn.apache.org/viewvc/hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java?rev=1501145&r1=1501144&r2=1501145&view=diff
==============================================================================
--- hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java (original)
+++ hive/branches/vectorization/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeDescUtils.java Tue Jul 9 09:07:35 2013
@@ -204,7 +204,7 @@ public class ExprNodeDescUtils {
return backtrack(column, parent, terminal);
}
if (source instanceof ExprNodeFieldDesc) {
- // field epression should be resolved
+ // field expression should be resolved
ExprNodeFieldDesc field = (ExprNodeFieldDesc) source.clone();
field.setDesc(backtrack(field.getDesc(), current, terminal));
return field;