You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by zs...@apache.org on 2008/12/23 00:23:33 UTC
svn commit: r728823 [1/3] - in /hadoop/hive/trunk: ./
ql/src/java/org/apache/hadoop/hive/ql/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/lib/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/ ql/src/java/org/apache/...
Author: zshao
Date: Mon Dec 22 15:23:32 2008
New Revision: 728823
URL: http://svn.apache.org/viewvc?rev=728823&view=rev
Log:
HIVE-186. Refactor code to use a single graph, nodeprocessor, dispatcher and rule abstraction. (Ashish Thusoo via zshao)
Added:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java (contents, props changed)
- copied, changed from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java
Removed:
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTEventDispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultASTProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultDispatcher.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/FunctionSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/GenMapRedWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseDriver.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PartitionPruner.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/PrintOpTreeProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TableSample.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TopoWalker.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/tools/LineageInfo.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
hadoop/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/tool/TestLineageInfo.java
hadoop/hive/trunk/ql/src/test/templates/TestParse.vm
hadoop/hive/trunk/ql/src/test/templates/TestParseNegative.vm
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Mon Dec 22 15:23:32 2008
@@ -26,6 +26,9 @@
IMPROVEMENTS
+ HIVE-186. Refactor code to use a single graph, nodeprocessor, dispatcher
+ and rule abstraction. (Ashish Thusoo via zshao)
+
HIVE-192. Add database name parameter to getTable and dropTable.
(Johan Oskarsson via zshao)
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Driver.java Mon Dec 22 15:23:32 2008
@@ -22,7 +22,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.*;
-import org.antlr.runtime.tree.CommonTree;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.commons.lang.StringUtils;
@@ -160,10 +160,10 @@
resStream = null;
pd = new ParseDriver();
- CommonTree tree = pd.parse(command);
+ ASTNode tree = pd.parse(command);
while((tree.getToken() == null) && (tree.getChildCount() > 0)) {
- tree = (CommonTree)tree.getChild(0);
+ tree = (ASTNode)tree.getChild(0);
}
sem = SemanticAnalyzerFactory.get(conf, tree);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java Mon Dec 22 15:23:32 2008
@@ -168,7 +168,7 @@
/**
* @return the name of the operator
*/
- public String getOperatorName() {
+ public String getName() {
return new String("FS");
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Mon Dec 22 15:23:32 2008
@@ -73,4 +73,13 @@
conditionInspectableObject.o.getClass().getName());
}
}
+
+ /**
+ * @return the name of the operator
+ */
+ @Override
+ public String getName() {
+ return new String("FIL");
+ }
+
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/GroupByOperator.java Mon Dec 22 15:23:32 2008
@@ -610,4 +610,12 @@
return colLists;
}
+
+ /**
+ * @return the name of the operator
+ */
+ @Override
+ public String getName() {
+ return new String("GBY");
+ }
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Operator.java Mon Dec 22 15:23:32 2008
@@ -21,7 +21,9 @@
import java.io.Serializable;
import java.util.HashMap;
import java.util.List;
+import java.util.Vector;
import java.util.Map;
+import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -36,7 +38,7 @@
/**
* Base operator implementation
**/
-public abstract class Operator <T extends Serializable> implements Serializable {
+public abstract class Operator <T extends Serializable> implements Serializable, Node {
// Bean methods
@@ -55,6 +57,23 @@
return childOperators;
}
+ /**
+ * Implements the getChildren function for the Node Interface.
+ */
+ public Vector<Node> getChildren() {
+
+ if (getChildOperators() == null) {
+ return null;
+ }
+
+ Vector<Node> ret_vec = new Vector<Node>();
+ for(Operator<? extends Serializable> op: getChildOperators()) {
+ ret_vec.add(op);
+ }
+
+ return ret_vec;
+ }
+
public void setParentOperators(List<Operator<? extends Serializable>> parentOperators) {
this.parentOperators = parentOperators;
}
@@ -288,10 +307,10 @@
}
/**
- * returns the name of the operator - specific subclasses can override the name as and when needed
+ * Implements the getName function for the Node Interface.
* @return the name of the operator
*/
- public String getOperatorName() {
+ public String getName() {
return new String("OP");
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java Mon Dec 22 15:23:32 2008
@@ -200,7 +200,7 @@
/**
* @return the name of the operator
*/
- public String getOperatorName() {
+ public String getName() {
return new String("RS");
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java Mon Dec 22 15:23:32 2008
@@ -87,4 +87,12 @@
}
forward(output, outputObjectInspector);
}
+
+ /**
+ * @return the name of the operator
+ */
+ @Override
+ public String getName() {
+ return new String("SEL");
+ }
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java Mon Dec 22 15:23:32 2008
@@ -52,7 +52,7 @@
* The operator name for this operator type. This is used to construct the rule for an operator
* @return the operator name
**/
- public String getOperatorName() {
+ public String getName() {
return new String("TS");
}
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultOpGraphWalker.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java Mon Dec 22 15:23:32 2008
@@ -16,9 +16,8 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
-import java.io.Serializable;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -26,39 +25,44 @@
import java.util.Set;
import java.util.Stack;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
+import org.apache.hadoop.hive.ql.exec.ScriptOperator;
+import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* base class for operator graph walker
* this class takes list of starting ops and walks them one by one. it maintains list of walked
* operators (dispatchedList) and a list of operators that are discovered but not yet dispatched
*/
-public abstract class DefaultOpGraphWalker implements OpGraphWalker {
+public class DefaultGraphWalker implements GraphWalker {
- List<Operator<? extends Serializable>> toWalk = new ArrayList<Operator<? extends Serializable>>();
- Set<Operator<? extends Serializable>> dispatchedList = new HashSet<Operator<? extends Serializable>>();
- Dispatcher dispatcher;
+ protected Stack<Node> opStack;
+ private List<Node> toWalk = new ArrayList<Node>();
+ private Set<Node> dispatchedList = new HashSet<Node>();
+ private Dispatcher dispatcher;
/**
* Constructor
* @param ctx graph of operators to walk
* @param disp dispatcher to call for each op encountered
*/
- public DefaultOpGraphWalker(Dispatcher disp) {
+ public DefaultGraphWalker(Dispatcher disp) {
this.dispatcher = disp;
- }
+ opStack = new Stack<Node>();
+ }
/**
* @return the toWalk
*/
- public List<Operator<? extends Serializable>> getToWalk() {
+ public List<Node> getToWalk() {
return toWalk;
}
/**
* @return the doneList
*/
- public Set<Operator<? extends Serializable>> getDispatchedList() {
+ public Set<Node> getDispatchedList() {
return dispatchedList;
}
@@ -68,25 +72,41 @@
* @param opStack stack of operators encountered
* @throws SemanticException
*/
- public void dispatch(Operator<? extends Serializable> op, Stack opStack) throws SemanticException {
- this.dispatcher.dispatch(op, opStack);
- this.dispatchedList.add(op);
+ public void dispatch(Node nd, Stack<Node> ndStack) throws SemanticException {
+ this.dispatcher.dispatch(nd, ndStack);
+ this.dispatchedList.add(nd);
}
/**
* starting point for walking
* @throws SemanticException
*/
- public void startWalking(Collection<Operator<? extends Serializable>> startOps) throws SemanticException {
- toWalk.addAll(startOps);
+ public void startWalking(Collection<Node> startNodes) throws SemanticException {
+ toWalk.addAll(startNodes);
while(toWalk.size() > 0)
walk(toWalk.remove(0));
}
/**
* walk the current operator and its descendants
- * @param op current operator in the graph
+ * @param nd current operator in the graph
* @throws SemanticException
*/
- public abstract void walk(Operator<? extends Serializable> op) throws SemanticException;
+ public void walk(Node nd) throws SemanticException {
+ opStack.push(nd);
+
+ if((nd.getChildren() == null)
+ || getDispatchedList().containsAll(nd.getChildren())) {
+ // all children are done or no need to walk the children
+ dispatch(nd, opStack);
+ opStack.pop();
+ return;
+ }
+ // move all the children to the front of queue
+ getToWalk().removeAll(nd.getChildren());
+ getToWalk().addAll(0, nd.getChildren());
+ // add self to the end of the queue
+ getToWalk().add(nd);
+ opStack.pop();
+ }
}
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultGraphWalker.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DefaultRuleDispatcher.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java Mon Dec 22 15:23:32 2008
@@ -16,21 +16,12 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
-import java.io.Serializable;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.lang.ClassNotFoundException;
-import java.util.List;
import java.util.Map;
import java.util.Stack;
-import java.lang.ClassNotFoundException;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext;
-import org.apache.hadoop.hive.ql.exec.OperatorFactory;
-import org.apache.hadoop.hive.ql.exec.OperatorFactory.opTuple;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Dispatches calls to relevant method in processor. The user registers various rules with the dispatcher, and
@@ -38,95 +29,54 @@
*/
public class DefaultRuleDispatcher implements Dispatcher {
- private Map<Rule, OperatorProcessor> opProcRules;
- private OperatorProcessorContext opProcCtx;
- private OperatorProcessor defaultProc;
+ private Map<Rule, NodeProcessor> procRules;
+ private NodeProcessorCtx procCtx;
+ private NodeProcessor defaultProc;
/**
* constructor
* @param defaultProc defualt processor to be fired if no rule matches
- * @param opp operator processor that handles actual processing of the node
- * @param opProcCtx operator processor context, which is opaque to the dispatcher
+ * @param rules operator processor that handles actual processing of the node
+ * @param procCtx operator processor context, which is opaque to the dispatcher
*/
- public DefaultRuleDispatcher(OperatorProcessor defaultProc,
- Map<Rule, OperatorProcessor> opp, OperatorProcessorContext opProcCtx) {
+ public DefaultRuleDispatcher(NodeProcessor defaultProc,
+ Map<Rule, NodeProcessor> rules, NodeProcessorCtx procCtx) {
this.defaultProc = defaultProc;
- this.opProcRules = opp;
- this.opProcCtx = opProcCtx;
+ this.procRules = rules;
+ this.procCtx = procCtx;
}
/**
* dispatcher function
- * @param op operator to process
- * @param opStack the operators encountered so far
+ * @param nd operator to process
+ * @param ndStack the operators encountered so far
* @throws SemanticException
*/
- public void dispatch(Operator<? extends Serializable> op, Stack<Operator<? extends Serializable>> opStack)
+ public void dispatch(Node nd, Stack<Node> ndStack)
throws SemanticException {
// find the firing rule
// find the rule from the stack specified
Rule rule = null;
int minCost = Integer.MAX_VALUE;
- for (Rule r : opProcRules.keySet()) {
- int cost = r.cost(opStack);
+ for (Rule r : procRules.keySet()) {
+ int cost = r.cost(ndStack);
if ((cost >= 0) && (cost <= minCost)) {
minCost = cost;
rule = r;
}
}
- OperatorProcessor proc;
+ NodeProcessor proc;
if (rule == null)
proc = defaultProc;
else
- proc = opProcRules.get(rule);
+ proc = procRules.get(rule);
- // If the processor has registered a process method for the particular operator, invoke it.
- // Otherwise implement the generic function, which would definitely be implemented
- for(opTuple opt : OperatorFactory.opvec) {
- if(opt.opClass.isInstance(op)) {
- Method pcall;
- try {
- pcall = proc.getClass().getMethod("process", opt.opClass,
- Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext"));
- pcall.invoke(proc, op, opProcCtx);
- return;
- } catch (SecurityException e) {
- assert false;
- } catch (NoSuchMethodException e) {
- assert false;
- } catch (IllegalArgumentException e) {
- assert false;
- } catch (IllegalAccessException e) {
- assert false;
- } catch (InvocationTargetException e) {
- throw new SemanticException(e.getTargetException());
- } catch (ClassNotFoundException e) {
- assert false;
- }
- }
- }
-
- try {
- // no method found - invoke the generic function
- Method pcall = proc.getClass().getMethod("process", Class.forName("org.apache.hadoop.hive.ql.exec.Operator"),
- Class.forName("org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext"));
- pcall.invoke(proc, ((Operator<? extends Serializable>)op), opProcCtx);
-
- } catch (SecurityException e) {
- assert false;
- } catch (NoSuchMethodException e) {
- assert false;
- } catch (IllegalArgumentException e) {
- assert false;
- } catch (IllegalAccessException e) {
- assert false;
- } catch (InvocationTargetException e) {
- throw new SemanticException(e.getTargetException());
- } catch (ClassNotFoundException e) {
- assert false;
- }
+ // Do nothing in case proc is null
+ if (proc != null)
+ // Call the process function
+ proc.process(nd, procCtx);
}
}
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/DefaultRuleDispatcher.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Dispatcher.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java Mon Dec 22 15:23:32 2008
@@ -16,12 +16,11 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
-import java.io.Serializable;
import java.util.Stack;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Dispatcher interface for Operators
@@ -31,10 +30,10 @@
/**
* dispatcher function
- * @param op operator to process
+ * @param nd operator to process
* @param Stack operator stack to process
* @throws SemanticException
*/
- public abstract void dispatch(Operator<? extends Serializable> op, Stack<Operator<? extends Serializable>> stack)
+ public abstract void dispatch(Node nd, Stack<Node> stack)
throws SemanticException;
}
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Dispatcher.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OpGraphWalker.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java Mon Dec 22 15:23:32 2008
@@ -16,25 +16,24 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
-import java.io.Serializable;
import java.util.Collection;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Interface for operator graph walker.
*/
-public interface OpGraphWalker {
+public interface GraphWalker {
/**
* starting point for walking.
*
- * @param startOps list of starting operators
+ * @param startNodes list of starting operators
* @throws SemanticException
*/
- public abstract void startWalking(Collection<Operator<? extends Serializable>> startOps)
+ public void startWalking(Collection<Node> startNodes)
throws SemanticException;
}
\ No newline at end of file
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/GraphWalker.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/OperatorProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java Mon Dec 22 15:23:32 2008
@@ -15,25 +15,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
-import java.io.Serializable;
-import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.optimizer.OperatorProcessorContext;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Base class for processing operators which is no-op. The specific processors can register their own context with
* the dispatcher.
*/
-public interface OperatorProcessor {
+public interface NodeProcessor {
/**
* generic process for all ops that don't have specific implementations
- * @param op operator to process
- * @param opProcCtx operator processor context
+ * @param nd operator to process
+ * @param procCtx operator processor context
* @throws SemanticException
*/
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext opProcCtx)
+ public void process(Node nd, NodeProcessorCtx procCtx)
throws SemanticException;
}
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessor.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/OperatorProcessorContext.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java Mon Dec 22 15:23:32 2008
@@ -16,10 +16,10 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.optimizer;
+package org.apache.hadoop.hive.ql.lib;
/**
* Operator Processor Context
*/
-public abstract class OperatorProcessorContext {
+public abstract class NodeProcessorCtx {
}
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/NodeProcessorCtx.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Rule.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java Mon Dec 22 15:23:32 2008
@@ -16,11 +16,10 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
import java.util.Stack;
-import java.io.Serializable;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Rule interface for Operators
@@ -32,7 +31,7 @@
* @return the cost of the rule - the lower the cost, the better the rule matches
* @throws SemanticException
*/
- public int cost(Stack<Operator<? extends Serializable>> stack) throws SemanticException;
+ public int cost(Stack<Node> stack) throws SemanticException;
/**
* @return the name of the rule - may be useful for debugging
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/Rule.java
------------------------------------------------------------------------------
svn:mergeinfo =
Copied: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java (from r728790, hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java)
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java?p2=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java&p1=hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java&r1=728790&r2=728823&rev=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/RuleRegExp.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java Mon Dec 22 15:23:32 2008
@@ -16,49 +16,45 @@
* limitations under the License.
*/
-package org.apache.hadoop.hive.ql.parse;
+package org.apache.hadoop.hive.ql.lib;
import java.util.Stack;
-import java.util.Iterator;
-import java.util.regex.Pattern;
import java.util.regex.Matcher;
-import java.io.Serializable;
+import java.util.regex.Pattern;
-import org.apache.hadoop.hive.ql.exec.Operator;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
- * Rule interface for Operators
- * Used in operator dispatching to dispatch process/visitor functions for operators
+ * Rule interface for Nodes
+ * Used in Node dispatching to dispatch process/visitor functions for Nodes
*/
public class RuleRegExp implements Rule {
private String ruleName;
- private String regExp;
private Pattern pattern;
/**
- * The rule specified by the regular expression. Note that, the regular expression is specified in terms of operator
- * name. For eg: TS.*RS -> means TableScan operator followed by anything any number of times followed by ReduceSink
+ * The rule specified by the regular expression. Note that, the regular expression is specified in terms of Node
+ * name. For eg: TS.*RS -> means TableScan Node followed by anything any number of times followed by ReduceSink
* @param ruleName name of the rule
* @param regExp regular expression for the rule
**/
public RuleRegExp(String ruleName, String regExp) {
this.ruleName = ruleName;
- this.regExp = regExp;
pattern = Pattern.compile(regExp);
}
/**
* This function returns the cost of the rule for the specified stack. Lower the cost, the better the rule is matched
- * @param stacl operator stack encountered so far
+ * @param stack Node stack encountered so far
* @return cost of the function
* @throws SemanticException
*/
- public int cost(Stack<Operator<? extends Serializable>> stack) throws SemanticException {
- int numElems = stack.size();
+ public int cost(Stack<Node> stack) throws SemanticException {
+ int numElems = (stack != null ? stack.size() : 0);
String name = new String();
for (int pos = numElems - 1; pos >= 0; pos--) {
- name = stack.get(pos).getOperatorName().concat(name);
+ name = stack.get(pos).getName() + "%" + name;
Matcher m = pattern.matcher(name);
if (m.matches())
return m.group().length();
@@ -68,7 +64,7 @@
}
/**
- * @return the name of the operator
+ * @return the name of the Node
**/
public String getName() {
return ruleName;
Propchange: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/RuleRegExp.java
------------------------------------------------------------------------------
svn:mergeinfo =
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPruner.java Mon Dec 22 15:23:32 2008
@@ -22,42 +22,34 @@
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Stack;
-
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
-import org.apache.hadoop.hive.ql.exec.ExtractOperator;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FilterOperator;
-import org.apache.hadoop.hive.ql.exec.GroupByOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.OperatorFactory;
-import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.exec.ScriptOperator;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.hive.ql.parse.DefaultDispatcher;
-import org.apache.hadoop.hive.ql.parse.Dispatcher;
-import org.apache.hadoop.hive.ql.parse.DefaultOpGraphWalker;
-import org.apache.hadoop.hive.ql.parse.OpGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.parse.OpParseContext;
-import org.apache.hadoop.hive.ql.parse.OperatorProcessor;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.QB;
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticAnalyzerFactory;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.plan.aggregationDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.exprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.groupByDesc;
-import org.apache.hadoop.hive.ql.plan.reduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.selectDesc;
-import org.apache.hadoop.hive.ql.parse.Rule;
/**
* Implementation of one of the rule-based optimization steps. ColumnPruner gets the current operator tree. The \
@@ -162,16 +154,31 @@
boolean done = true;
// generate pruned column list for all relevant operators
- ColumnPrunerProcessor cpp = new ColumnPrunerProcessor(opToParseCtxMap);
- Dispatcher disp = new DefaultDispatcher(cpp);
- OpGraphWalker ogw = new ColumnPrunerWalker(disp);
- ogw.startWalking(pGraphContext.getTopOps().values());
+ ColumnPrunerProcCtx cppCtx = new ColumnPrunerProcCtx(opToParseCtxMap);
+
+ // create a walker which walks the tree in a DFS manner while maintaining the operator stack. The dispatcher
+ // generates the plan from the operator tree
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+ opRules.put(new RuleRegExp("R1", "FIL%"), ColumnPrunerProcFactory.getFilterProc());
+ opRules.put(new RuleRegExp("R2", "GBY%"), ColumnPrunerProcFactory.getGroupByProc());
+ opRules.put(new RuleRegExp("R3", "RS%"), ColumnPrunerProcFactory.getReduceSinkProc());
+ opRules.put(new RuleRegExp("R4", "SEL%"), ColumnPrunerProcFactory.getSelectProc());
+
+ // The dispatcher fires the processor corresponding to the closest matching rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(ColumnPrunerProcFactory.getDefaultProc(), opRules, cppCtx);
+ GraphWalker ogw = new ColumnPrunerWalker(disp);
+
+ // Create a list of topop nodes
+ ArrayList<Node> topNodes = new ArrayList<Node>();
+ topNodes.addAll(pGraphContext.getTopOps().values());
+ ogw.startWalking(topNodes);
// create a new select operator if any of input tables' columns can be pruned
for (String alias_id : pGraphContext.getTopOps().keySet()) {
Operator<? extends Serializable> topOp = pGraphContext.getTopOps().get(alias_id);
- List<String> colNames = cpp.getPrunedColList(topOp);
+ List<String> colNames = cppCtx.getPrunedColList(topOp);
+
// do we need to push a SELECT - all the columns of the table are not used
if (pushSelect(topOp, colNames)) {
topOp.setChildOperators(null);
@@ -198,157 +205,11 @@
}
return pGraphContext;
}
-
- /**
- * Column pruner processor
- **/
- public static class ColumnPrunerProcessor implements OperatorProcessor {
- private Map<Operator<? extends Serializable>,List<String>> prunedColLists =
- new HashMap<Operator<? extends Serializable>, List<String>>();
- private HashMap<Operator<? extends Serializable>, OpParseContext> opToParseCtxMap;
-
- public ColumnPrunerProcessor(HashMap<Operator<? extends Serializable>, OpParseContext> opToParseContextMap) {
- this.opToParseCtxMap = opToParseContextMap;
- }
-
- /**
- * @return the prunedColLists
- */
- public List<String> getPrunedColList(Operator<? extends Serializable> op) {
- return prunedColLists.get(op);
- }
-
- private List<String> genColLists(Operator<? extends Serializable> curOp) throws SemanticException {
- List<String> colList = new ArrayList<String>();
- if(curOp.getChildOperators() != null) {
- for(Operator<? extends Serializable> child: curOp.getChildOperators())
- colList = Utilities.mergeUniqElems(colList, prunedColLists.get(child));
- }
- return colList;
- }
-
- public void process(FilterOperator op, OperatorProcessorContext ctx) throws SemanticException {
- exprNodeDesc condn = op.getConf().getPredicate();
- // get list of columns used in the filter
- List<String> cl = condn.getCols();
- // merge it with the downstream col list
- prunedColLists.put(op, Utilities.mergeUniqElems(genColLists(op), cl));
- }
-
- public void process(GroupByOperator op, OperatorProcessorContext ctx) throws SemanticException {
- List<String> colLists = new ArrayList<String>();
- groupByDesc conf = op.getConf();
- ArrayList<exprNodeDesc> keys = conf.getKeys();
- for (exprNodeDesc key : keys)
- colLists = Utilities.mergeUniqElems(colLists, key.getCols());
-
- ArrayList<aggregationDesc> aggrs = conf.getAggregators();
- for (aggregationDesc aggr : aggrs) {
- ArrayList<exprNodeDesc> params = aggr.getParameters();
- for (exprNodeDesc param : params)
- colLists = Utilities.mergeUniqElems(colLists, param.getCols());
- }
-
- prunedColLists.put(op, colLists);
- }
-
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext ctx) throws SemanticException {
- prunedColLists.put(op, genColLists(op));
- }
-
- public void process(ReduceSinkOperator op, OperatorProcessorContext ctx) throws SemanticException {
- RowResolver redSinkRR = opToParseCtxMap.get(op).getRR();
- reduceSinkDesc conf = op.getConf();
- List<Operator<? extends Serializable>> childOperators = op.getChildOperators();
- List<Operator<? extends Serializable>> parentOperators = op.getParentOperators();
- List<String> childColLists = new ArrayList<String>();
-
- for(Operator<? extends Serializable> child: childOperators)
- childColLists = Utilities.mergeUniqElems(childColLists, prunedColLists.get(child));
-
- List<String> colLists = new ArrayList<String>();
- ArrayList<exprNodeDesc> keys = conf.getKeyCols();
- for (exprNodeDesc key : keys)
- colLists = Utilities.mergeUniqElems(colLists, key.getCols());
-
- if ((childOperators.size() == 1) && (childOperators.get(0) instanceof JoinOperator)) {
- assert parentOperators.size() == 1;
- Operator<? extends Serializable> par = parentOperators.get(0);
- RowResolver parRR = opToParseCtxMap.get(par).getRR();
- RowResolver childRR = opToParseCtxMap.get(childOperators.get(0)).getRR();
-
- for (String childCol : childColLists) {
- String [] nm = childRR.reverseLookup(childCol);
- ColumnInfo cInfo = redSinkRR.get(nm[0],nm[1]);
- if (cInfo != null) {
- cInfo = parRR.get(nm[0], nm[1]);
- if (!colLists.contains(cInfo.getInternalName()))
- colLists.add(cInfo.getInternalName());
- }
- }
- }
- else {
- // Reduce Sink contains the columns needed - no need to aggregate from children
- ArrayList<exprNodeDesc> vals = conf.getValueCols();
- for (exprNodeDesc val : vals)
- colLists = Utilities.mergeUniqElems(colLists, val.getCols());
- }
-
- prunedColLists.put(op, colLists);
- }
-
- public void process(SelectOperator op, OperatorProcessorContext ctx) throws SemanticException {
- List<String> cols = new ArrayList<String>();
-
- if(op.getChildOperators() != null) {
- for(Operator<? extends Serializable> child: op.getChildOperators()) {
- // If one of my children is a FileSink or Script, return all columns.
- // Without this break, a bug in ReduceSink to Extract edge column pruning will manifest
- // which should be fixed before remove this
- if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator)) {
- prunedColLists.put(op, getColsFromSelectExpr(op));
- return;
- }
- cols = Utilities.mergeUniqElems(cols, prunedColLists.get(child));
- }
- }
-
- selectDesc conf = op.getConf();
- if (conf.isSelectStar() && !cols.isEmpty()) {
- // The input to the select does not matter. Go over the expressions
- // and return the ones which have a marked column
- prunedColLists.put(op, getSelectColsFromChildren(op, cols));
- return;
- }
- prunedColLists.put(op, getColsFromSelectExpr(op));
- }
-
- private List<String> getColsFromSelectExpr(SelectOperator op) {
- List<String> cols = new ArrayList<String>();
- selectDesc conf = op.getConf();
- ArrayList<exprNodeDesc> exprList = conf.getColList();
- for (exprNodeDesc expr : exprList)
- cols = Utilities.mergeUniqElems(cols, expr.getCols());
- return cols;
- }
-
- private List<String> getSelectColsFromChildren(SelectOperator op, List<String> colList) {
- List<String> cols = new ArrayList<String>();
- selectDesc conf = op.getConf();
- ArrayList<exprNodeDesc> selectExprs = conf.getColList();
-
- for (String col : colList) {
- // col is the internal name i.e. position within the expression list
- exprNodeDesc expr = selectExprs.get(Integer.parseInt(col));
- cols = Utilities.mergeUniqElems(cols, expr.getCols());
- }
- return cols;
- }
- }
+
/**
* Walks the op tree in post order fashion (skips selects with file sink or script op children)
*/
- public static class ColumnPrunerWalker extends DefaultOpGraphWalker {
+ public static class ColumnPrunerWalker extends DefaultGraphWalker {
public ColumnPrunerWalker(Dispatcher disp) {
super(disp);
@@ -358,30 +219,33 @@
* Walk the given operator
*/
@Override
- public void walk(Operator<? extends Serializable> op) throws SemanticException {
+ public void walk(Node nd) throws SemanticException {
boolean walkChildren = true;
+ opStack.push(nd);
// no need to go further down for a select op with a file sink or script child
// since all cols are needed for these ops
- if(op instanceof SelectOperator) {
- for(Operator<? extends Serializable> child: op.getChildOperators()) {
+ if(nd instanceof SelectOperator) {
+ for(Node child: nd.getChildren()) {
if ((child instanceof FileSinkOperator) || (child instanceof ScriptOperator))
walkChildren = false;
}
}
- if((op.getChildOperators() == null)
- || getDispatchedList().containsAll(op.getChildOperators())
+ if((nd.getChildren() == null)
+ || getDispatchedList().containsAll(nd.getChildren())
|| !walkChildren) {
// all children are done or no need to walk the children
- dispatch(op, null);
+ dispatch(nd, opStack);
+ opStack.pop();
return;
}
// move all the children to the front of queue
- getToWalk().removeAll(op.getChildOperators());
- getToWalk().addAll(0, op.getChildOperators());
+ getToWalk().removeAll(nd.getChildren());
+ getToWalk().addAll(0, nd.getChildren());
// add self to the end of the queue
- getToWalk().add(op);
+ getToWalk().add(nd);
+ opStack.pop();
}
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRFileSink1.java Mon Dec 22 15:23:32 2008
@@ -23,27 +23,29 @@
import java.io.Serializable;
import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.parse.OperatorProcessor;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.plan.mapredWork;
-import org.apache.hadoop.hive.ql.metadata.*;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.parse.SemanticException;
/**
* Processor for the rule - table scan followed by reduce sink
*/
-public class GenMRFileSink1 implements OperatorProcessor {
+public class GenMRFileSink1 implements NodeProcessor {
public GenMRFileSink1() {
}
/**
* File Sink Operator encountered
- * @param op the file sink operator encountered
+ * @param nd the file sink operator encountered
* @param opProcCtx context
*/
- public void process(FileSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException {
+ public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+ FileSinkOperator op = (FileSinkOperator)nd;
GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
boolean ret = false;
@@ -81,13 +83,4 @@
}
}
}
-
- /**
- * @param op the operator encountered
- * @param opProcCtx context
- */
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext opProcCtx) throws SemanticException {
- // should never be called
- assert false;
- }
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMROperator.java Mon Dec 22 15:23:32 2008
@@ -23,26 +23,28 @@
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.parse.SemanticException;
-import org.apache.hadoop.hive.ql.parse.OperatorProcessor;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
/**
* Processor for the rule - no specific rule fired
*/
-public class GenMROperator implements OperatorProcessor {
+public class GenMROperator implements NodeProcessor {
public GenMROperator() {
}
/**
* Reduce Scan encountered
- * @param op the reduce sink operator encountered
- * @param opProcCtx context
+ * @param nd the reduce sink operator encountered
+ * @param procCtx context
*/
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext opProcCtx) throws SemanticException {
- GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
+ public void process(Node nd, NodeProcessorCtx procCtx) throws SemanticException {
+ GenMRProcContext ctx = (GenMRProcContext)procCtx;
Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
- mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
+ mapCurrCtx.put((Operator<? extends Serializable>)nd, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
}
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRProcContext.java Mon Dec 22 15:23:32 2008
@@ -45,6 +45,7 @@
import org.apache.hadoop.hive.ql.plan.partitionDesc;
import org.apache.hadoop.hive.ql.plan.fileSinkDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.exec.Utilities;
@@ -54,7 +55,7 @@
* Processor Context for creating map reduce task. Walk the tree in a DFS manner and process the nodes. Some state is
* maintained about the current nodes visited so far.
*/
-public class GenMRProcContext extends OperatorProcessorContext {
+public class GenMRProcContext extends NodeProcessorCtx {
/**
* GenMapRedCtx is used to keep track of the current state.
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink1.java Mon Dec 22 15:23:32 2008
@@ -18,54 +18,35 @@
package org.apache.hadoop.hive.ql.optimizer;
-import java.util.List;
-import java.util.ArrayList;
import java.util.Map;
import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.Set;
-import java.util.Stack;
import java.io.Serializable;
-import java.io.File;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
-import org.apache.hadoop.hive.ql.exec.TableScanOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskFactory;
-import org.apache.hadoop.hive.ql.exec.OperatorFactory;
import org.apache.hadoop.hive.ql.plan.mapredWork;
-import org.apache.hadoop.hive.ql.plan.reduceSinkDesc;
-import org.apache.hadoop.hive.ql.plan.tableDesc;
-import org.apache.hadoop.hive.ql.plan.partitionDesc;
-import org.apache.hadoop.hive.ql.plan.fileSinkDesc;
-import org.apache.hadoop.hive.ql.plan.PlanUtils;
-import org.apache.hadoop.hive.ql.metadata.*;
-import org.apache.hadoop.hive.ql.exec.Utilities;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.parse.OperatorProcessor;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
/**
* Processor for the rule - table scan followed by reduce sink
*/
-public class GenMRRedSink1 implements OperatorProcessor {
+public class GenMRRedSink1 implements NodeProcessor {
public GenMRRedSink1() {
}
/**
* Reduce Scan encountered
- * @param op the reduce sink operator encountered
+ * @param nd the reduce sink operator encountered
* @param opProcCtx context
*/
- public void process(ReduceSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException {
+ public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+ ReduceSinkOperator op = (ReduceSinkOperator)nd;
GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
@@ -100,13 +81,4 @@
mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
}
- /**
- * Reduce Scan encountered
- * @param op the reduce sink operator encountered
- * @param opProcCtx context
- */
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext opProcCtx) throws SemanticException {
- // should never be called
- assert false;
- }
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRRedSink2.java Mon Dec 22 15:23:32 2008
@@ -23,32 +23,28 @@
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
-import org.apache.hadoop.hive.ql.exec.JoinOperator;
import org.apache.hadoop.hive.ql.exec.Task;
-import org.apache.hadoop.hive.ql.exec.TaskFactory;
-import org.apache.hadoop.hive.ql.exec.OperatorFactory;
-import org.apache.hadoop.hive.ql.plan.mapredWork;
-import org.apache.hadoop.hive.ql.plan.reduceSinkDesc;
-import org.apache.hadoop.hive.ql.metadata.*;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.ql.parse.OperatorProcessor;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
/**
* Processor for the rule - reduce sink followed by reduce sink
*/
-public class GenMRRedSink2 implements OperatorProcessor {
+public class GenMRRedSink2 implements NodeProcessor {
public GenMRRedSink2() {
}
/**
* Reduce Scan encountered
- * @param op the reduce sink operator encountered
+ * @param nd the reduce sink operator encountered
* @param opProcCtx context
*/
- public void process(ReduceSinkOperator op, OperatorProcessorContext opProcCtx) throws SemanticException {
+ public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+ ReduceSinkOperator op = (ReduceSinkOperator)nd;
GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
@@ -75,14 +71,4 @@
mapCurrCtx.put(op, new GenMapRedCtx(ctx.getCurrTask(), ctx.getCurrTopOp(), ctx.getCurrAliasId()));
}
- /**
- * Reduce Scan encountered
- * @param op the operator encountered
- * @param opProcCtx context
- */
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext opProcCtx) throws SemanticException {
- // should never be called
- assert false;
- }
-
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java Mon Dec 22 15:23:32 2008
@@ -22,12 +22,12 @@
import java.util.Map;
import org.apache.hadoop.hive.ql.exec.Operator;
-import org.apache.hadoop.hive.ql.parse.OperatorProcessor;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
-import org.apache.hadoop.hive.ql.plan.mapredWork;
-import org.apache.hadoop.hive.ql.metadata.*;
+import org.apache.hadoop.hive.ql.lib.Node;
+import org.apache.hadoop.hive.ql.lib.NodeProcessor;
+import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.optimizer.GenMRProcContext.GenMapRedCtx;
@@ -35,16 +35,17 @@
/**
* Processor for the rule - table scan
*/
-public class GenMRTableScan1 implements OperatorProcessor {
+public class GenMRTableScan1 implements NodeProcessor {
public GenMRTableScan1() {
}
/**
* Table Sink encountered
- * @param op the table sink operator encountered
+ * @param nd the table sink operator encountered
* @param opProcCtx context
*/
- public void process(TableScanOperator op, OperatorProcessorContext opProcCtx) throws SemanticException {
+ public void process(Node nd, NodeProcessorCtx opProcCtx) throws SemanticException {
+ TableScanOperator op = (TableScanOperator)nd;
GenMRProcContext ctx = (GenMRProcContext)opProcCtx;
ParseContext parseCtx = ctx.getParseCtx();
Map<Operator<? extends Serializable>, GenMapRedCtx> mapCurrCtx = ctx.getMapCurrCtx();
@@ -67,14 +68,5 @@
assert false;
}
- /**
- * Table Sink encountered
- * @param op the table sink operator encountered
- * @param opProcCtx context
- */
- public void process(Operator<? extends Serializable> op, OperatorProcessorContext opProcCtx) throws SemanticException {
- // should never be called
- assert false;
- }
}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEvent.java Mon Dec 22 15:23:32 2008
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-/**
- * Enumeration that encapsulates the various event types that are seen
- * while processing the parse tree (in an implementation of the ParseTreeProcessor).
- * These event types are used to register the different event processors with
- * the parse tree processor.
- *
- */
-public enum ASTEvent {
-
- /**
- * Query event
- */
- QUERY("QUERY"),
-
- /**
- * Union
- */
- UNION("UNION"),
-
- /**
- * Source Table (table in the from clause)
- */
- SRC_TABLE("SRC_TABLE"),
-
- /**
- * Any type of Destination (this fires for hdfs directory, local directory and table)
- */
- DESTINATION("DESTINATION"),
-
- /**
- * Select clause
- */
- SELECT_CLAUSE("SELECT_CLAUSE"),
-
- /**
- * Join clause
- */
- JOIN_CLAUSE("JOIN_CLAUSE"),
-
- /**
- * Where clause
- */
- WHERE_CLAUSE("WHERE_CLAUSE"),
-
- /**
- * CLusterby clause
- */
- CLUSTERBY_CLAUSE("CLUSTERBY_CLAUSE"),
-
- /**
- * Group by clause
- */
- GROUPBY_CLAUSE("GROUPBY_CLAUSE"),
-
- /**
- * Limit clause
- */
- LIMIT_CLAUSE("LIMIT_CLAUSE"),
-
- /**
- * Subquery
- */
- SUBQUERY("SUBQUERY");
-
- /**
- * The name of the event (string representation of the event)
- */
- private final String name;
-
- /**
- * Constructs the event
- *
- * @param name The name(String representation of the event)
- */
- ASTEvent(String name) {
- this.name = name;
- }
-
- /**
- * String representation of the event
- *
- * @return String
- */
- @Override
- public String toString() {
- return name;
- }
-}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventDispatcher.java Mon Dec 22 15:23:32 2008
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-import org.antlr.runtime.tree.CommonTree;
-
-/**
- * Dispatches ParseTreeEvent to the appropriate ParseTreeEventProcessor
- */
-public interface ASTEventDispatcher {
-
- /**
- * Registers the event processor with the event
- *
- * @param evt The parse tree event
- * @param evt_p The associated parse tree event processor
- */
- void register(ASTEvent evt, ASTEventProcessor evt_p);
-
- /**
- * Dispatches the parse tree event to a registered event processor
- *
- * @param evt The parse tree event to dispatch
- * @param pt The parse subtree to dispatch to the event processor
- */
- void dispatch(ASTEvent evt, CommonTree pt);
-}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTEventProcessor.java Mon Dec 22 15:23:32 2008
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-import org.antlr.runtime.tree.CommonTree;
-
-/**
- * Interface that a parse tree event processor needs to implement. Classes implementing
- * this interface and registered with the ParseTreeProcessor are called by the later
- * when associated ParseTreeEvents are processed
- *
- */
-public interface ASTEventProcessor {
-
- /**
- * Processes the parse subtree corresponding to the event
- *
- * @param pt The parse subtree to process
- */
- public void process(CommonTree pt);
-}
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ASTProcessor.java Mon Dec 22 15:23:32 2008
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.parse;
-
-import org.antlr.runtime.tree.CommonTree;
-
-/**
- * Interface that a parse tree processor needs to implement
- */
-public interface ASTProcessor {
-
- /**
- * Sets the event dispatcher for the processors
- *
- * @param dispatcher The parse tree event dispatcher
- */
- void setDispatcher(ASTEventDispatcher dispatcher);
-
- /**
- * Processes the parse tree and calls the registered event processors
- * for the associated parse tree events
- *
- * @param pt The parse tree to process
- */
- void process(CommonTree pt);
-}
-
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java?rev=728823&r1=728822&r2=728823&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java Mon Dec 22 15:23:32 2008
@@ -25,8 +25,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.antlr.runtime.tree.CommonTree;
-
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.metadata.*;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -66,9 +64,9 @@
}
}
- public abstract void analyzeInternal(CommonTree ast, Context ctx) throws SemanticException;
+ public abstract void analyzeInternal(ASTNode ast, Context ctx) throws SemanticException;
- public void analyze(CommonTree ast, Context ctx) throws SemanticException {
+ public void analyze(ASTNode ast, Context ctx) throws SemanticException {
scratchDir = ctx.getScratchDir();
analyzeInternal(ast, ctx);
}
@@ -241,7 +239,7 @@
public HashMap<String, String> partSpec;
public Partition partHandle;
- public tableSpec(Hive db, CommonTree ast, boolean forceCreatePartition) throws SemanticException {
+ public tableSpec(Hive db, ASTNode ast, boolean forceCreatePartition) throws SemanticException {
assert(ast.getToken().getType() == HiveParser.TOK_TAB);
int childIndex = 0;
@@ -254,10 +252,10 @@
// get partition metadata if partition specified
if (ast.getChildCount() == 2) {
childIndex = 1;
- CommonTree partspec = (CommonTree) ast.getChild(1);
+ ASTNode partspec = (ASTNode) ast.getChild(1);
partSpec = new LinkedHashMap<String, String>();
for (int i = 0; i < partspec.getChildCount(); ++i) {
- CommonTree partspec_val = (CommonTree) partspec.getChild(i);
+ ASTNode partspec_val = (ASTNode) partspec.getChild(i);
String val = stripQuotes(partspec_val.getChild(1).getText());
partSpec.put(unescapeIdentifier(partspec_val.getChild(0).getText()), val);
}