You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/08 23:25:54 UTC
svn commit: r654629 [2/4] - in /incubator/pig/branches/types: ./
src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/parser/
src/org/apache/pig/impl/logicalLayer/schema/ src/org/apache/pig/impl/logic...
Added: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java?rev=654629&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java Thu May 8 14:25:22 2008
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class LORegexp extends ExpressionOperator {
+ private static final long serialVersionUID = 2L;
+
+ /**
+ * The expression and the column to be projected.
+ */
+ private ExpressionOperator mOperand;
+ private String mRegexp;
+ private static Log log = LogFactory.getLog(LORegexp.class);
+
+ /**
+ *
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
+ * @param exp
+ * the expression which might contain the column to project
+ * @param projection
+ * the list of columns to project
+ */
+ public LORegexp(LogicalPlan plan, OperatorKey key,
+ ExpressionOperator operand, String regexp) {
+ super(plan, key);
+ mOperand = operand;
+ mRegexp = regexp;
+ }
+
+ public ExpressionOperator getOperand() {
+ return mOperand;
+ }
+
+ public String getRegexp() {
+ return mRegexp;
+ }
+
+ @Override
+ public String name() {
+ return "Project " + mKey.scope + "-" + mKey.id;
+ }
+
+ @Override
+ public boolean supportsMultipleInputs() {
+ return false;
+ }
+
+ @Override
+ public Schema getSchema() {
+ return mSchema;
+ }
+
+ @Override
+ public void visit(LOVisitor v) throws VisitorException {
+ v.visit(this);
+ }
+
+}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java Thu May 8 14:25:22 2008
@@ -26,19 +26,26 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOSort extends LogicalOperator {
private static final long serialVersionUID = 2L;
- private List<Integer> mSortCols;
+ private LogicalOperator mInput;
private List<Boolean> mAscCols;
- private LOUserFunc mSortFunc;
+ private String mSortFunc;
+ private boolean mIsStar = false;
+ private List<LogicalPlan> mSortColPlans;
+ private static Log log = LogFactory.getLog(LOSort.class);
/**
* @param plan
* LogicalPlan this operator is a part of.
* @param key
* OperatorKey for this operator
+ * @param input
+ * Input to sort
* @param sortCols
* Array of column numbers that will be used for sorting data.
* @param ascCols
@@ -47,33 +54,40 @@
* this array is null, then all columns will be sorted ascending.
* @param sorFunc
* the user defined sorting function
- * @param rp
- * Requested level of parallelism to be used in the sort.
*/
- public LOSort(LogicalPlan plan,
- OperatorKey key,
- List<Integer> sortCols,
- List<Boolean> ascCols,
- LOUserFunc sortFunc,
- int rp) {
- super(plan, key, rp);
- mSortCols = sortCols;
+ public LOSort(LogicalPlan plan, OperatorKey key, LogicalOperator input,
+ List<LogicalPlan> sortColPlans, List<Boolean> ascCols, String sortFunc) {
+ super(plan, key);
+ mInput = input;
+ mSortColPlans = sortColPlans;
mAscCols = ascCols;
mSortFunc = sortFunc;
}
- public List<Integer> getSortCols() {
- return mSortCols;
+ public LogicalOperator getInput() {
+ return mInput;
+ }
+
+ public List<LogicalPlan> getSortColPlans() {
+ return mSortColPlans;
}
public List<Boolean> getAscendingCols() {
return mAscCols;
}
- public LOUserFunc getUserFunc() {
+ public String getUserFunc() {
return mSortFunc;
}
-
+
+ public void setUserFunc(String func) {
+ mSortFunc = func;
+ }
+
+ public void setStar(boolean b) {
+ mIsStar = b;
+ }
+
@Override
public String name() {
return "SORT " + mKey.scope + "-" + mKey.id;
@@ -83,10 +97,10 @@
public Schema getSchema() throws FrontendException {
if (!mIsSchemaComputed && (null == mSchema)) {
// get our parent's schema
- Collection<LogicalOperator> s = mPlan.getSuccessors(this);
+ Collection<LogicalOperator> s = mPlan.getPredecessors(this);
try {
LogicalOperator op = s.iterator().next();
- if(null == op) {
+ if (null == op) {
throw new FrontendException("Could not find operator in plan");
}
mSchema = op.getSchema();
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java Thu May 8 14:25:22 2008
@@ -27,40 +27,52 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOSplit extends LogicalOperator {
private static final long serialVersionUID = 2L;
- private Map<String, ExpressionOperator> mOutputs;
+ private Map<String, LogicalPlan> mCondPlans;
+ private ArrayList<LogicalOperator> mOutputs;
+ private static Log log = LogFactory.getLog(LOSplit.class);
/**
* @param plan
* LogicalPlan this operator is a part of.
* @param key
* OperatorKey for this operator
- * @param rp
- * Requested level of parallelism to be used in the sort.
- * @param aliases
+ * @param outputs
* list of aliases that are the output of the split
* @param conditions
* list of conditions for the split
*/
- public LOSplit(LogicalPlan plan, OperatorKey key, int rp,
- Map<String, ExpressionOperator> outputs) {
- super(plan, key, rp);
+ public LOSplit(LogicalPlan plan, OperatorKey key,
+ ArrayList<LogicalOperator> outputs,
+ Map<String, LogicalPlan> condPlans) {
+ super(plan, key);
mOutputs = outputs;
+ mCondPlans = condPlans;
}
- public Collection<ExpressionOperator> getConditions() {
- return mOutputs.values();
+ public List<LogicalOperator> getOutputs() {
+ return mOutputs;
+ }
+
+ public Collection<LogicalPlan> getConditionPlans() {
+ return mCondPlans.values();
}
public Set<String> getOutputAliases() {
- return mOutputs.keySet();
+ return mCondPlans.keySet();
}
- public void addOutputAlias(String output, ExpressionOperator cond) {
- mOutputs.put(output, cond);
+ public void addOutputAlias(String output, LogicalPlan cond) {
+ mCondPlans.put(output, cond);
+ }
+
+ public void addOutput(LogicalOperator lOp) {
+ mOutputs.add(lOp);
}
@Override
Added: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=654629&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java Thu May 8 14:25:22 2008
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.impl.logicalLayer;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.VisitorException;
+
+
+public class LOSplitOutput extends LogicalOperator {
+ private static final long serialVersionUID = 2L;
+
+ protected int mIndex;
+
+ /**
+ * @param plan
+ * LogicalPlan this operator is a part of.
+ * @param key
+ * OperatorKey for this operator
+ * @param outputs
+ * list of aliases that are the output of the split
+ * @param conditions
+ * list of conditions for the split
+ */
+ public LOSplitOutput(LogicalPlan plan, OperatorKey key, int index) {
+ super(plan, key);
+ this.mIndex = index;
+ }
+
+ @Override
+ public String name() {
+ return "SplitOutput " + mKey.scope + "-" + mKey.id;
+ }
+
+ @Override
+ public Schema getSchema() throws FrontendException{
+ if (!mIsSchemaComputed && (null == mSchema)) {
+ // get our parent's schema
+ Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+ try {
+ LogicalOperator op = s.iterator().next();
+ if (null == op) {
+ throw new FrontendException("Could not find operator in plan");
+ }
+ mSchema = s.iterator().next().getSchema();
+ mIsSchemaComputed = true;
+ } catch (FrontendException fe) {
+ mSchema = null;
+ mIsSchemaComputed = false;
+ throw fe;
+ }
+ }
+ return mSchema;
+ }
+
+ public void visit(LOVisitor v) throws VisitorException{
+ v.visit(this);
+ }
+
+ @Override
+ public boolean supportsMultipleInputs() {
+ return false;
+ }
+
+ @Override
+ public boolean supportsMultipleOutputs() {
+ return false;
+ }
+
+ public int getReadFrom() {
+ return mIndex;
+ }
+}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java Thu May 8 14:25:22 2008
@@ -23,33 +23,35 @@
import java.util.List;
import java.util.Map;
-import org.apache.pig.StoreFunc; // import org.apache.pig.impl.PigContext;
+import org.apache.pig.StoreFunc;
+import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOStore extends LogicalOperator {
private static final long serialVersionUID = 2L;
private FileSpec mOutputFile;
private StoreFunc mStoreFunc;
+ private static Log log = LogFactory.getLog(LOStore.class);
/**
* @param plan
* LogicalPlan this operator is a part of.
* @param key
* OperatorKey for this operator
- * @param rp
- * Requested level of parallelism to be used in the sort.
* @param outputFileSpec
* the file to be stored
* @param storeFunc
* the store function, pre-defined or user defined
*/
- public LOStore(LogicalPlan plan, OperatorKey key, int rp,
+ public LOStore(LogicalPlan plan, OperatorKey key,
FileSpec outputFileSpec) throws IOException {
- super(plan, key, rp);
+ super(plan, key);
mOutputFile = outputFileSpec;
@@ -58,13 +60,13 @@
// HExecutionEngine which in turn is completely commented out
// Also remove the commented out import org.apache.pig.impl.PigContext
- /*
- * try { mStoreFunc = (StoreFunc)
- * PigContext.instantiateFuncFromSpec(outputFileSpec.getFuncSpec()); }
- * catch (Exception e) { IOException ioe = new
- * IOException(e.getMessage()); ioe.setStackTrace(e.getStackTrace());
- * throw ioe; }
- */
+ try {
+ mStoreFunc = (StoreFunc) PigContext.instantiateFuncFromSpec(outputFileSpec.getFuncSpec());
+ } catch (Exception e) {
+ IOException ioe = new IOException(e.getMessage());
+ ioe.setStackTrace(e.getStackTrace());
+ throw ioe;
+ }
}
public FileSpec getOutputFile() {
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSubtract.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSubtract.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSubtract.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSubtract.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOSubtract extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOSubtract.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOSubtract(LogicalPlan plan, OperatorKey k, int rp,
+ public LOSubtract(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java Thu May 8 14:25:22 2008
@@ -22,22 +22,35 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOUnion extends LogicalOperator {
private static final long serialVersionUID = 2L;
+ private ArrayList<LogicalOperator> mInputs;
+ private static Log log = LogFactory.getLog(LOUnion.class);
/**
* @param plan
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
+ * @param inputs
+ * List of operators that are input to the union
*/
- public LOUnion(LogicalPlan plan, OperatorKey k, int rp) {
- super(plan, k, rp);
+ public LOUnion(LogicalPlan plan, OperatorKey k,
+ ArrayList<LogicalOperator> inputs) {
+ super(plan, k);
+ mInputs = inputs;
+ }
+
+ public List<LogicalOperator> getInputs() {
+ return mInputs;
+ }
+
+ public void addInput(LogicalOperator input) {
+ mInputs.add(input);
}
@Override
@@ -46,6 +59,8 @@
// TODO FIX
// The schema merge operation needs to be implemented in
// order to compute the schema of the union
+ //If schemas are the same then return one of the schemas else
+ //return null
}
return mSchema;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java Thu May 8 14:25:22 2008
@@ -26,6 +26,7 @@
public class LOUserFunc extends ExpressionOperator {
private static final long serialVersionUID = 2L;
+ private String mFuncName;
private List<ExpressionOperator> mArgs;
/**
@@ -33,18 +34,25 @@
* LogicalPlan this operator is a part of.
* @param k
* OperatorKey for this operator.
+ * @param funcName
+ * name of the user defined function.
* @param args
* List of expressions that form the arguments for this function.
* @param returnType
* return type of this function.
*/
- public LOUserFunc(LogicalPlan plan, OperatorKey k,
+ public LOUserFunc(LogicalPlan plan, OperatorKey k, String funcName,
List<ExpressionOperator> args, byte returnType) {
super(plan, k, -1);
+ mFuncName = funcName;
mArgs = args;
mType = returnType;
}
+ public String getFuncName() {
+ return mFuncName;
+ }
+
public List<ExpressionOperator> getArguments() {
return mArgs;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java Thu May 8 14:25:22 2008
@@ -20,10 +20,15 @@
import java.util.List;
import java.util.Iterator;
import java.util.Set;
+import java.util.Map;
+import java.util.ArrayList;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.PlanWalker;
+import org.apache.pig.impl.plan.DepthFirstWalker;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.plan.MultiMap;
/**
* A visitor mechanism for navigating and operating on a tree of Logical
@@ -43,7 +48,8 @@
* myEval.visit(v); WRONG: LOEval myEval; MyVisitor v; v.visitEval(myEval);
* These methods are only public to make them accessible to the LO* objects.
*/
-abstract public class LOVisitor extends PlanVisitor<LogicalOperator, LogicalPlan> {
+abstract public class LOVisitor extends
+ PlanVisitor<LogicalOperator, LogicalPlan> {
public LOVisitor(LogicalPlan plan,
PlanWalker<LogicalOperator, LogicalPlan> walker) {
@@ -110,9 +116,18 @@
*/
protected void visit(LOCogroup cg) throws VisitorException {
// Visit each of the inputs of cogroup.
- Iterator<ExpressionOperator> i = cg.getGroupByCols().iterator();
- while (i.hasNext()) {
- i.next().visit(this);
+ MultiMap<LogicalOperator, LogicalPlan> mapGByPlans = cg.getGroupByPlans();
+ for(LogicalOperator op: cg.getInputs()) {
+ for(LogicalPlan lp: mapGByPlans.get(op)) {
+ if (null != lp) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+ }
}
}
@@ -124,10 +139,14 @@
*/
protected void visit(LOGenerate g) throws VisitorException {
// Visit each of generates projection elements.
- Iterator<ExpressionOperator> i = g.getProjections().iterator();
- while (i.hasNext()) {
- i.next().visit(this);
- }
+ for(LogicalPlan lp: g.getGeneratePlans()) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
}
/**
@@ -138,7 +157,14 @@
*/
protected void visit(LOSort s) throws VisitorException {
// Visit the sort function
- s.getUserFunc().visit(this);
+ for(LogicalPlan lp: s.getSortColPlans()) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
}
/**
@@ -149,7 +175,12 @@
*/
protected void visit(LOFilter filter) throws VisitorException {
// Visit the condition for the filter followed by the input
- filter.getCondition().visit(this);
+ PlanWalker w = new DependencyOrderWalker(filter.getComparisonPlan());
+ pushWalker(w);
+ for(LogicalOperator logicalOp: filter.getComparisonPlan().getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
}
/**
@@ -160,10 +191,16 @@
*/
protected void visit(LOSplit split) throws VisitorException {
// Visit each of split's conditions
- Iterator<ExpressionOperator> i = split.getConditions().iterator();
- while (i.hasNext()) {
- i.next().visit(this);
- }
+ for(LogicalPlan lp: split.getConditionPlans()) {
+ if (null != lp) {
+ PlanWalker w = new DependencyOrderWalker(lp);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: lp.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
+ }
+ }
}
/**
@@ -173,11 +210,14 @@
* @throws VisitorException
*/
protected void visit(LOForEach forEach) throws VisitorException {
- // Visit the operators that are part of the foreach
- Iterator<LogicalOperator> i = forEach.getOperators().iterator();
- while (i.hasNext()) {
- i.next().visit(this);
- }
+ // Visit the operators that are part of the foreach plan
+ LogicalPlan plan = forEach.getForEachPlan();
+ PlanWalker w = new DependencyOrderWalker(plan);
+ pushWalker(w);
+ for(LogicalOperator logicalOp: plan.getRoots()) {
+ logicalOp.visit(this);
+ }
+ popWalker();
}
/**
@@ -196,11 +236,12 @@
}
/**
- * @param binCond the logical binCond operator that has to be visited
+ * @param binCond
+ * the logical binCond operator that has to be visited
* @throws VisitorException
*/
protected void visit(LOBinCond binCond) throws VisitorException {
- /**
+ /*
* Visit the conditional expression followed by the left hand operator
* and the right hand operator respectively
*/
@@ -222,7 +263,17 @@
cast.getExpression().visit(this);
}
-
+ /**
+ *
+ * @param regexp
+ * the logical regexp operator that has to be visited
+ * @throws ParseException
+ */
+ protected void visit(LORegexp regexp) throws VisitorException {
+ // Visit the operand of the regexp
+ regexp.getOperand().visit(this);
+ }
+
protected void visit(LOLoad load) throws VisitorException{
@@ -236,4 +287,12 @@
}
+ protected void visit(LOProject project) throws VisitorException {
+ // Visit the operand of the project as long as the sentinel is false
+
+ if(!project.getSentinel()) {
+ project.getExpression().visit(this);
+ }
+ }
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Thu May 8 14:25:22 2008
@@ -28,6 +28,9 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.Operator;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* Parent for all Logical operators.
@@ -44,10 +47,10 @@
* A boolean variable to remember if the schema has been computed
*/
protected boolean mIsSchemaComputed = false;
-
+
/**
- * Datatype of this output of this operator. Operators start out with data type
- * set to UNKNOWN, and have it set for them by the type checker.
+ * Datatype of this output of this operator. Operators start out with data
+ * type set to UNKNOWN, and have it set for them by the type checker.
*/
protected byte mType = DataType.UNKNOWN;
@@ -59,7 +62,7 @@
/**
* Name of the record set that results from this operator.
*/
- protected String mAlias;
+ protected String mAlias;
/**
* Logical plan that this operator is a part of.
@@ -67,25 +70,40 @@
protected LogicalPlan mPlan;
/**
+ * A boolean variable to remember if input has to be flattened Used only in
+ * the context of generate
+ */
+ //private boolean mIsFlatten = false;
+
+ private static Log log = LogFactory.getLog(LogicalOperator.class);
+
+ /**
* Equivalent to LogicalOperator(k, 0).
- * @param plan Logical plan this operator is a part of.
- * @param k Operator key to assign to this node.
+ *
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
*/
public LogicalOperator(LogicalPlan plan, OperatorKey k) {
this(plan, k, -1);
}
/**
- * @param plan Logical plan this operator is a part of.
- * @param - k Operator key to assign to this node.
- * @param = rp degree of requested parallelism with which to execute this node.
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param -
+ * k Operator key to assign to this node.
+ * @param =
+ * rp degree of requested parallelism with which to execute this
+ * node.
*/
public LogicalOperator(LogicalPlan plan, OperatorKey k, int rp) {
super(k);
mPlan = plan;
mRequestedParallelism = rp;
}
-
+
/**
* Get the operator key for this operator.
*/
@@ -94,12 +112,14 @@
}
/**
- * Set the output schema for this operator. If a schema already
- * exists, an attempt will be made to reconcile it with this new
- * schema.
- * @param schema Schema to set.
- * @throws ParseException if there is already a schema and the existing
- * schema cannot be reconciled with this new schema.
+ * Set the output schema for this operator. If a schema already exists, an
+ * attempt will be made to reconcile it with this new schema.
+ *
+ * @param schema
+ * Schema to set.
+ * @throws ParseException
+ * if there is already a schema and the existing schema cannot
+ * be reconciled with this new schema.
*/
public final void setSchema(Schema schema) throws ParseException {
// In general, operators don't generate their schema until they're
@@ -109,8 +129,14 @@
} catch (FrontendException ioe) {
// It's fine, it just means we don't have a schema yet.
}
- if (mSchema == null) mSchema = schema;
- else mSchema.reconcile(schema);
+ if (mSchema == null) {
+ log.debug("Operator schema is null; Setting it to new schema");
+ mSchema = schema;
+ } else {
+ log.debug("Reconciling schema");
+ log.debug("mSchema: " + mSchema + " schema: " + schema);
+ mSchema.reconcile(schema);
+ }
}
/**
@@ -119,9 +145,11 @@
public abstract Schema getSchema() throws FrontendException;
/**
- * Set the type of this operator. This should only be called by the type
+ * Set the type of this operator. This should only be called by the type
* checking routines.
- * @param type - Type to set this operator to.
+ *
+ * @param type -
+ * Type to set this operator to.
*/
final public void setType(byte t) {
mType = t;
@@ -133,7 +161,7 @@
public byte getType() {
return mType;
}
-
+
public String getAlias() {
return mAlias;
}
@@ -153,7 +181,7 @@
@Override
public String toString() {
StringBuffer msg = new StringBuffer();
-
+
msg.append("(Name: " + name() + " Operator Key: " + mKey + ")");
return msg.toString();
@@ -161,8 +189,11 @@
/**
* Given a schema, reconcile it with our existing schema.
- * @param schema Schema to reconcile with the existing.
- * @throws ParseException if the reconciliation is not possible.
+ *
+ * @param schema
+ * Schema to reconcile with the existing.
+ * @throws ParseException
+ * if the reconciliation is not possible.
*/
protected void reconcileSchema(Schema schema) throws ParseException {
if (mSchema == null) {
@@ -184,4 +215,14 @@
*/
public abstract void visit(LOVisitor v) throws VisitorException;
+ /*
+ public boolean isFlatten() {
+ return mIsFlatten;
+ }
+
+ public void setFlatten(boolean b) {
+ mIsFlatten = b;
+ }
+ */
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Thu May 8 14:25:22 2008
@@ -35,8 +35,14 @@
super();
}
+ /*
+ public LogicalPlan(Map<OperatorKey, LogicalOperator> keys) {
+ super(keys);
+ }
+ */
+
public void explain(OutputStream out) {
- // TODO FIX
+ // TODO
/*
LOVisitor lprinter = new LOPrinter(new PrintStream(out));
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java Thu May 8 14:25:22 2008
@@ -18,7 +18,7 @@
package org.apache.pig.impl.logicalLayer;
// TODO FIX
-/*
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.util.Map;
@@ -26,7 +26,6 @@
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.logicalLayer.parser.ParseException;
import org.apache.pig.impl.logicalLayer.parser.QueryParser;
-*/
/**
@@ -35,7 +34,7 @@
*/
public class LogicalPlanBuilder {
// TODO FIX
- /*
+
public static ClassLoader classloader = LogicalPlanBuilder.class.getClassLoader();
private PigContext pigContext;
public LogicalPlanBuilder(PigContext pigContext) {
@@ -45,11 +44,13 @@
public LogicalPlan parse(String scope,
String query,
Map<String, LogicalPlan> aliases,
- Map<OperatorKey, LogicalOperator> opTable)
+ Map<OperatorKey, LogicalOperator> opTable,
+ Map<String, LogicalOperator> aliasOp)
throws IOException, ParseException {
ByteArrayInputStream in = new ByteArrayInputStream(query.getBytes());
- QueryParser parser = new QueryParser(in, pigContext, scope, aliases, opTable);
+ //QueryParser parser = new QueryParser(in, pigContext, scope, aliases, opTable);
+ QueryParser parser = new QueryParser(in, pigContext, scope, aliases, opTable, aliasOp);
return parser.Parse();
}
- */
+
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java Thu May 8 14:25:22 2008
@@ -21,6 +21,8 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.data.DataType;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* This abstract class represents the logical Unary Expression Operator The
@@ -30,6 +32,7 @@
public abstract class UnaryExpressionOperator extends ExpressionOperator {
private static final long serialVersionUID = 2L;
private ExpressionOperator mOperand; // operand
+ private static Log log = LogFactory.getLog(UnaryExpressionOperator.class);
/**
* @param plan
@@ -50,6 +53,22 @@
mOperand = operand;
}
+ /**
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
+ * @param operand
+ * ExpressionOperator the left hand side operand
+ * @param operator
+ * LogicalExperssion the actual operator
+ */
+ public UnaryExpressionOperator(LogicalPlan plan, OperatorKey k,
+ ExpressionOperator operand) {
+ super(plan, k);
+ mOperand = operand;
+ }
+
public ExpressionOperator getOperand() {
return mOperand;
}