You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/11/14 23:56:51 UTC
svn commit: r714180 [1/2] - in /hadoop/pig/branches/types: ./
src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/parser/
src/org/apache/pig/impl/logicalLayer/validators/
src/org/apache/pig/impl/plan/ test/org/apache/pig/test/ te...
Author: olga
Date: Fri Nov 14 14:56:50 2008
New Revision: 714180
URL: http://svn.apache.org/viewvc?rev=714180&view=rev
Log:
PIG-512: Expressions in foreach lead to errors
Added:
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloneHelper.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloner.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/RemoveRedundantOperators.java
Modified:
hadoop/pig/branches/types/CHANGES.txt
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLimit.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
hadoop/pig/branches/types/src/org/apache/pig/impl/plan/Operator.java
hadoop/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
hadoop/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
hadoop/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan1.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan2.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan3.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Arithmetic.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/BinCond.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Cogroup.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Comparison.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/ComplexForeach.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Distinct.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Generate.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull2.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Limit.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Sort.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split2.gld
hadoop/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Union.gld
Modified: hadoop/pig/branches/types/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/CHANGES.txt?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/CHANGES.txt (original)
+++ hadoop/pig/branches/types/CHANGES.txt Fri Nov 14 14:56:50 2008
@@ -312,3 +312,5 @@
via olgan)
PIG-525: make sure cast for udf parameters works (olgan)
+
+ PIG-512: Expressions in foreach lead to errors (sms via olgan)
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java Fri Nov 14 14:56:50 2008
@@ -100,4 +100,15 @@
return true;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ BinaryExpressionOperator binExOpClone = (BinaryExpressionOperator)super.clone();
+ return binExOpClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java Fri Nov 14 14:56:50 2008
@@ -132,6 +132,18 @@
}
}
+ /**
+ * @see org.apache.pig.impl.plan.LogicalOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ ExpressionOperator exOpClone = (ExpressionOperator)super.clone();
+ if(mFieldSchema != null)
+ exOpClone.mFieldSchema = this.mFieldSchema.clone();
+ return exOpClone;
+ }
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java Fri Nov 14 14:56:50 2008
@@ -72,6 +72,10 @@
return mRhsOp;
}
+ public void setCond(ExpressionOperator cond) {
+ mCond = cond;
+ }
+
public void setLhsOp(ExpressionOperator op) {
mLhsOp = op ;
}
@@ -117,4 +121,15 @@
return true;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOBinCond clone = (LOBinCond)super.clone();
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java Fri Nov 14 14:56:50 2008
@@ -56,6 +56,10 @@
return mExpr;
}
+ public void setExpression(ExpressionOperator expr) {
+ mExpr = expr;
+ }
+
@Override
public void visit(LOVisitor v) throws VisitorException {
v.visit(this);
@@ -93,4 +97,18 @@
mLoadFuncSpec = loadFuncSpec;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOCast clone = (LOCast)super.clone();
+ if(mLoadFuncSpec != null) {
+ clone.mLoadFuncSpec = mLoadFuncSpec.clone();
+ }
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java Fri Nov 14 14:56:50 2008
@@ -24,6 +24,7 @@
import java.util.Set;
import java.util.HashSet;
import java.util.HashMap;
+import java.util.Iterator;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.plan.OperatorKey;
@@ -79,6 +80,10 @@
return mGroupByPlans;
}
+ public void setGroupByPlans(MultiMap<LogicalOperator, LogicalPlan> groupByPlans) {
+ mGroupByPlans = groupByPlans;
+ }
+
public boolean[] getInner() {
return mIsInner;
}
@@ -473,4 +478,35 @@
fs.setParent(null, op);
}
}
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LogicalOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+
+ // first start with LogicalOperator clone
+ LOCogroup cogroupClone = (LOCogroup)super.clone();
+
+ // create deep copy of other cogroup specific members
+ cogroupClone.mIsInner = new boolean[mIsInner.length];
+ for (int i = 0; i < mIsInner.length; i++) {
+ cogroupClone.mIsInner[i] = mIsInner[i];
+ }
+
+ cogroupClone.mGroupByPlans = new MultiMap<LogicalOperator, LogicalPlan>();
+ for (Iterator<LogicalOperator> it = mGroupByPlans.keySet().iterator(); it.hasNext();) {
+ LogicalOperator relOp = it.next();
+ Collection<LogicalPlan> values = mGroupByPlans.get(relOp);
+ for (Iterator<LogicalPlan> planIterator = values.iterator(); planIterator.hasNext();) {
+ LogicalPlanCloneHelper lpCloneHelper = new LogicalPlanCloneHelper(planIterator.next());
+ cogroupClone.mGroupByPlans.put(relOp, lpCloneHelper.getClonedPlan());
+ }
+ }
+
+ return cogroupClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java Fri Nov 14 14:56:50 2008
@@ -92,4 +92,15 @@
return true;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOConst clone = (LOConst)super.clone();
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java Fri Nov 14 14:56:50 2008
@@ -103,4 +103,16 @@
public byte getType() {
return DataType.BAG ;
}
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LogicalOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LODistinct distinctClone = (LODistinct)super.clone();
+ return distinctClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java Fri Nov 14 14:56:50 2008
@@ -112,5 +112,17 @@
super.unsetSchema();
}
+ /**
+ * @see org.apache.pig.impl.plan.Operator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOFilter filterClone = (LOFilter)super.clone();
+ LogicalPlanCloneHelper lpCloner = new LogicalPlanCloneHelper(mComparisonPlan);
+ filterClone.mComparisonPlan = lpCloner.getClonedPlan();
+ return filterClone;
+ }
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java Fri Nov 14 14:56:50 2008
@@ -395,4 +395,40 @@
super.unsetSchema();
}
+ /**
+ * @see org.apache.pig.impl.plan.Operator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ // Do generic LogicalOperator cloning
+ LOForEach forEachClone = (LOForEach)super.clone();
+
+ // create deep copies of attributes specific to foreach
+ if(mFlatten != null) {
+ forEachClone.mFlatten = new ArrayList<Boolean>();
+ for (Iterator<Boolean> it = mFlatten.iterator(); it.hasNext();) {
+ forEachClone.mFlatten.add(new Boolean(it.next()));
+ }
+ }
+
+ if(mForEachPlans != null) {
+ forEachClone.mForEachPlans = new ArrayList<LogicalPlan>();
+ for (Iterator<LogicalPlan> it = mForEachPlans.iterator(); it.hasNext();) {
+ LogicalPlanCloneHelper lpCloneHelper = new LogicalPlanCloneHelper(it.next());
+ forEachClone.mForEachPlans.add(lpCloneHelper.getClonedPlan());
+ }
+ }
+
+ if(mUserDefinedSchema != null) {
+ forEachClone.mUserDefinedSchema = new ArrayList<Schema>();
+ for (Iterator<Schema> it = mUserDefinedSchema.iterator(); it.hasNext();) {
+ Schema s = it.next();
+ forEachClone.mUserDefinedSchema.add(s != null ? s.clone() : null);
+ }
+ }
+ return forEachClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java Fri Nov 14 14:56:50 2008
@@ -151,4 +151,38 @@
v.visit(this);
}
+ /* (non-Javadoc)
+ * @see org.apache.pig.impl.plan.Operator#clone()
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ // Do generic LogicalOperator cloning
+ LOGenerate generateClone = (LOGenerate)super.clone();
+
+ // create deep copies of attributes specific to foreach
+ if(mFlatten != null) {
+ generateClone.mFlatten = new ArrayList<Boolean>();
+ for (Iterator<Boolean> it = mFlatten.iterator(); it.hasNext();) {
+ generateClone.mFlatten.add(new Boolean(it.next()));
+ }
+ }
+
+ if(mGeneratePlans != null) {
+ generateClone.mGeneratePlans = new ArrayList<LogicalPlan>();
+ for (Iterator<LogicalPlan> it = mGeneratePlans.iterator(); it.hasNext();) {
+ LogicalPlanCloneHelper lpCloneHelper = new LogicalPlanCloneHelper(it.next());
+ generateClone.mGeneratePlans.add(lpCloneHelper.getClonedPlan());
+ }
+ }
+
+ if(mUserDefinedSchema != null) {
+ generateClone.mUserDefinedSchema = new ArrayList<Schema>();
+ for (Iterator<Schema> it = mUserDefinedSchema.iterator(); it.hasNext();) {
+ Schema s = it.next();
+ generateClone.mUserDefinedSchema.add(s != null ? s.clone() : null);
+ }
+ }
+ return generateClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLimit.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLimit.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLimit.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLimit.java Fri Nov 14 14:56:50 2008
@@ -81,4 +81,15 @@
{
return new LOLimit(mPlan, OperatorKey.genOpKey(mKey.scope), mLimit);
}
+
+ /**
+ * @see org.apache.pig.impl.plan.Operator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOLimit limitClone = (LOLimit)super.clone();
+ return limitClone;
+ }
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java Fri Nov 14 14:56:50 2008
@@ -77,6 +77,10 @@
return mMap;
}
+ public void setMap(ExpressionOperator map) {
+ mMap = map;
+ }
+
public Object getLookUpKey() {
return mMapKey;
}
@@ -120,4 +124,21 @@
v.visit(this);
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOMapLookup clone = (LOMapLookup)super.clone();
+
+ // deep copy project specific attributes
+ if(mValueSchema != null) {
+ clone.mValueSchema = mValueSchema.clone();
+ }
+
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java Fri Nov 14 14:56:50 2008
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import java.util.Iterator;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -407,4 +408,22 @@
return sb.toString() ;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOProject clone = (LOProject)super.clone();
+
+ // deep copy project specific attributes
+ clone.mProjection = new ArrayList<Integer>();
+ for (Iterator<Integer> it = mProjection.iterator(); it.hasNext();) {
+ clone.mProjection.add(new Integer(it.next()));
+ }
+
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java Fri Nov 14 14:56:50 2008
@@ -21,6 +21,7 @@
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Iterator;
import org.apache.pig.FuncSpec;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -169,4 +170,35 @@
public byte getType() {
return DataType.BAG ;
}
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LogicalOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOSort clone = (LOSort) super.clone();
+
+ // deep copy sort related attributes
+ if(mAscCols != null) {
+ clone.mAscCols = new ArrayList<Boolean>();
+ for (Iterator<Boolean> it = mAscCols.iterator(); it.hasNext();) {
+ clone.mAscCols.add(new Boolean(it.next()));
+ }
+ }
+
+ if(mSortFunc != null)
+ clone.mSortFunc = mSortFunc.clone();
+
+ if(mSortColPlans != null) {
+ clone.mSortColPlans = new ArrayList<LogicalPlan>();
+ for (Iterator<LogicalPlan> it = mSortColPlans.iterator(); it.hasNext();) {
+ LogicalPlanCloneHelper lpCloneHelper = new LogicalPlanCloneHelper(it.next());
+ clone.mSortColPlans.add(lpCloneHelper.getClonedPlan());
+ }
+ }
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java Fri Nov 14 14:56:50 2008
@@ -56,6 +56,10 @@
return mOutputs;
}
+ public void setOutputs(ArrayList<LogicalOperator> outputs) {
+ mOutputs = outputs;
+ }
+
public void addOutput(LogicalOperator lOp) {
mOutputs.add(lOp);
}
@@ -105,4 +109,14 @@
return DataType.BAG;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LogicalOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOSplit splitClone = (LOSplit)super.clone();
+ return splitClone;
+ }
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java Fri Nov 14 14:56:50 2008
@@ -109,4 +109,17 @@
super.unsetSchema();
}
+ /**
+ * @see org.apache.pig.impl.plan.Operator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOSplitOutput splitOutputClone = (LOSplitOutput)super.clone();
+ LogicalPlanCloneHelper lpCloner = new LogicalPlanCloneHelper(mCondPlan);
+ splitOutputClone.mCondPlan = lpCloner.getClonedPlan();
+ return splitOutputClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java Fri Nov 14 14:56:50 2008
@@ -113,4 +113,15 @@
return DataType.BAG;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LogicalOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOUnion unionClone = (LOUnion)super.clone();
+ return unionClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java Fri Nov 14 14:56:50 2008
@@ -18,6 +18,7 @@
package org.apache.pig.impl.logicalLayer;
import java.util.List;
+import java.util.ArrayList;
import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
@@ -63,6 +64,10 @@
return mArgs;
}
+ public void setArguments(List<ExpressionOperator> args) {
+ mArgs = args;
+ }
+
@Override
public boolean supportsMultipleInputs() {
return true;
@@ -132,4 +137,17 @@
mArgs = args;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LOUserFunc clone = (LOUserFunc)super.clone();
+ //note that mFuncSpec cannot be null in LOUserFunc
+ clone.mFuncSpec = mFuncSpec.clone();
+ return clone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java Fri Nov 14 14:56:50 2008
@@ -274,4 +274,17 @@
return true;
}
+ /**
+ * @see org.apache.pig.impl.plan.Operator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ LogicalOperator loClone = (LogicalOperator)super.clone();
+ if(mSchema != null)
+ loClone.mSchema = this.mSchema.clone();
+ return loClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java Fri Nov 14 14:56:50 2008
@@ -27,9 +27,11 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Collection;
import org.apache.pig.impl.plan.OperatorPlan;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.plan.PlanException;
public class LogicalPlan extends OperatorPlan<LogicalOperator> {
private static final long serialVersionUID = 2L;
@@ -75,4 +77,59 @@
// return baos.toString();
// }
// }
+
+ /**
+ * Do not use the clone method directly. Use {@link LogicalPlanCloner} instead.
+ */
+ @Override
+ public LogicalPlan clone() throws CloneNotSupportedException {
+ LogicalPlan clone = new LogicalPlan();
+
+ // Get all the nodes in this plan, and clone them. As we make
+ // clones, create a map between clone and original. Then walk the
+ // connections in this plan and create equivalent connections in the
+ // clone.
+ Map<LogicalOperator, LogicalOperator> matches =
+ //new HashMap<LogicalOperator, LogicalOperator>(mOps.size());
+ LogicalPlanCloneHelper.mOpToCloneMap;
+ for (LogicalOperator op : mOps.keySet()) {
+ try {
+ LogicalOperator c = (LogicalOperator)op.clone();
+ clone.add(c);
+ matches.put(op, c);
+ } catch (CloneNotSupportedException cnse) {
+ cnse.printStackTrace();
+ throw cnse;
+ }
+ }
+
+ // Build the edges
+ for (LogicalOperator op : mToEdges.keySet()) {
+ LogicalOperator cloneTo = matches.get(op);
+ if (cloneTo == null) {
+ String msg = new String("Unable to find clone for op "
+ + op.name());
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+ Collection<LogicalOperator> fromOps = mToEdges.get(op);
+ for (LogicalOperator fromOp : fromOps) {
+ LogicalOperator cloneFrom = matches.get(fromOp);
+ if (cloneFrom == null) {
+ String msg = new String("Unable to find clone for op "
+ + fromOp.name());
+ log.error(msg);
+ throw new RuntimeException(msg);
+ }
+ try {
+ clone.connect(cloneFrom, cloneTo);
+ } catch (PlanException pe) {
+ throw new RuntimeException(pe);
+ }
+ }
+ }
+
+ return clone;
+ }
+
}
Added: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloneHelper.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloneHelper.java?rev=714180&view=auto
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloneHelper.java (added)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloneHelper.java Fri Nov 14 14:56:50 2008
@@ -0,0 +1,477 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Map;
+
+import org.apache.pig.impl.plan.PlanException;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.util.MultiMap;
+
+/**
+ * LogicalPlanCloneHelper implements a visitor mechanism to clone a logical plan
+ * and then patch up the connections held within the operators of the logical plan.
+ * This class should not be used for cloning the logical plan. Use {@link LogicalPlanCloner}
+ * instead.
+ */
+public class LogicalPlanCloneHelper extends LOVisitor {
+
+ public static Map<LogicalOperator, LogicalOperator> mOpToCloneMap;
+ private LogicalPlan mOriginalPlan;
+
+ /**
+ * @param plan logical plan to be cloned
+ * @param walker walker used for traversing the logical plan
+ */
+ public LogicalPlanCloneHelper(LogicalPlan plan) throws CloneNotSupportedException {
+ super(plan, new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
+ mOriginalPlan = plan;
+ //LOVisitor does not have a default constructor and super needs to be the first
+ //statement in the constructor. As a result, mPlan and mCurrentWalker are being
+ //re-initialized here
+ mPlan = plan.clone();
+ mCurrentWalker = new DependencyOrderWalker<LogicalOperator, LogicalPlan>(mPlan);
+ }
+
+ /**
+ * @param plan
+ * @param walker
+ * @param origCloneMap the lookup table used for tracking operators cloned in the plan
+ */
+ public LogicalPlanCloneHelper(LogicalPlan plan,
+ Map<LogicalOperator, LogicalOperator> origCloneMap) throws CloneNotSupportedException {
+ super(plan, new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
+ mOpToCloneMap = origCloneMap;
+ mOriginalPlan = plan;
+ //LOVisitor does not have a default constructor and super needs to be the first
+ //statement in the constructor. As a result, mPlan and mCurrentWalker are being
+ //re-initialized here
+ mPlan = plan.clone();
+ mCurrentWalker = new DependencyOrderWalker<LogicalOperator, LogicalPlan>(mPlan);
+ }
+
+ public LogicalPlan getClonedPlan() throws CloneNotSupportedException {
+
+ // set the "mPlan" member of all the Logical operators
+ // in the cloned plan to the cloned plan
+ PlanSetter ps = new PlanSetter(mPlan);
+ try {
+ ps.visit();
+ //patch up the connections
+ this.visit();
+
+ } catch (VisitorException e) {
+ CloneNotSupportedException cnse = new CloneNotSupportedException("Unable to set plan correctly during cloning");
+ cnse.initCause(e);
+ throw cnse;
+ }
+
+ return mPlan;
+ }
+
+ public static void resetState() {
+ mOpToCloneMap.clear();
+ }
+
+ @Override
+ public void visit(BinaryExpressionOperator binOp) {
+ ExpressionOperator lhsOperandClone = (ExpressionOperator)mOpToCloneMap.get(binOp.getLhsOperand());
+ if(lhsOperandClone != null) {
+ binOp.setLhsOperand(lhsOperandClone);
+ }
+ ExpressionOperator rhsOperandClone = (ExpressionOperator)mOpToCloneMap.get(binOp.getRhsOperand());
+ if(rhsOperandClone != null) {
+ binOp.setRhsOperand(rhsOperandClone);
+ }
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOAdd)
+ */
+ @Override
+ public void visit(LOAdd op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOAnd)
+ */
+ @Override
+ public void visit(LOAnd binOp) throws VisitorException {
+ this.visit((BinaryExpressionOperator)binOp);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOBinCond)
+ */
+ @Override
+ protected void visit(LOBinCond binCond) throws VisitorException {
+ ExpressionOperator condClone = (ExpressionOperator)mOpToCloneMap.get(binCond.getCond());
+ if(condClone != null) {
+ binCond.setCond(condClone);
+ }
+
+ ExpressionOperator lhsOpClone = (ExpressionOperator)mOpToCloneMap.get(binCond.getLhsOp());
+ if(lhsOpClone != null) {
+ binCond.setLhsOp(lhsOpClone);
+ }
+
+ ExpressionOperator rhsOpClone = (ExpressionOperator)mOpToCloneMap.get(binCond.getRhsOp());
+ if(rhsOpClone != null) {
+ binCond.setRhsOp(rhsOpClone);
+ }
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOCast)
+ */
+ @Override
+ protected void visit(LOCast cast) throws VisitorException {
+ ExpressionOperator exprClone = (ExpressionOperator)mOpToCloneMap.get(cast.getExpression());
+ if(exprClone != null) {
+ cast.setExpression(exprClone);
+ }
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOCogroup)
+ */
+ @Override
+ protected void visit(LOCogroup cg) throws VisitorException {
+ MultiMap<LogicalOperator, LogicalPlan> groupByPlans = cg.getGroupByPlans();
+ MultiMap<LogicalOperator, LogicalPlan> groupByPlansClone = new MultiMap<LogicalOperator, LogicalPlan>();
+ for(LogicalOperator cgInput: groupByPlans.keySet()) {
+ LogicalOperator cgInputClone = mOpToCloneMap.get(cgInput);
+ if(cgInputClone != null) {
+ groupByPlansClone.put(cgInputClone, groupByPlans.get(cgInput));
+ } else {
+ groupByPlansClone.put(cgInput, groupByPlans.get(cgInput));
+ }
+ }
+ cg.setGroupByPlans(groupByPlansClone);
+ super.visit(cg);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOConst)
+ */
+ @Override
+ protected void visit(LOConst constant) throws VisitorException {
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOCross)
+ */
+ @Override
+ protected void visit(LOCross cs) throws VisitorException {
+ super.visit(cs);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LODistinct)
+ */
+ @Override
+ protected void visit(LODistinct dt) throws VisitorException {
+ super.visit(dt);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LODivide)
+ */
+ @Override
+ public void visit(LODivide op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOEqual)
+ */
+ @Override
+ public void visit(LOEqual op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOFilter)
+ */
+ @Override
+ protected void visit(LOFilter filter) throws VisitorException {
+ super.visit(filter);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOForEach)
+ */
+ @Override
+ protected void visit(LOForEach forEach) throws VisitorException {
+ super.visit(forEach);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOGenerate)
+ */
+ @Override
+ protected void visit(LOGenerate g) throws VisitorException {
+ super.visit(g);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LogicalOperator)
+ */
+ @Override
+ protected void visit(LogicalOperator op) throws VisitorException {
+ super.visit(op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOGreaterThan)
+ */
+ @Override
+ public void visit(LOGreaterThan op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOGreaterThanEqual)
+ */
+ @Override
+ public void visit(LOGreaterThanEqual op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOIsNull)
+ */
+ @Override
+ public void visit(LOIsNull uniOp) throws VisitorException {
+ this.visit((UnaryExpressionOperator) uniOp);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOLesserThan)
+ */
+ @Override
+ public void visit(LOLesserThan op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOLesserThanEqual)
+ */
+ @Override
+ public void visit(LOLesserThanEqual op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOLimit)
+ */
+ @Override
+ protected void visit(LOLimit limOp) throws VisitorException {
+ super.visit(limOp);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOLoad)
+ */
+ @Override
+ protected void visit(LOLoad load) throws VisitorException {
+ //TODO
+ //LOLoad cloning is not implemented
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOMapLookup)
+ */
+ @Override
+ public void visit(LOMapLookup op) throws VisitorException {
+ ExpressionOperator mapClone = (ExpressionOperator)mOpToCloneMap.get(op.getMap());
+ if(mapClone != null) {
+ op.setMap(mapClone);
+ }
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOMod)
+ */
+ @Override
+ public void visit(LOMod op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOMultiply)
+ */
+ @Override
+ public void visit(LOMultiply op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LONegative)
+ */
+ @Override
+ public void visit(LONegative op) throws VisitorException {
+ this.visit((UnaryExpressionOperator) op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LONot)
+ */
+ @Override
+ public void visit(LONot uniOp) throws VisitorException {
+ this.visit((UnaryExpressionOperator) uniOp);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LONotEqual)
+ */
+ @Override
+ public void visit(LONotEqual op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOOr)
+ */
+ @Override
+ public void visit(LOOr binOp) throws VisitorException {
+ this.visit((BinaryExpressionOperator)binOp);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOProject)
+ */
+ @Override
+ protected void visit(LOProject project) throws VisitorException {
+ LogicalOperator projectInputClone = mOpToCloneMap.get(project.getExpression());
+ if(projectInputClone != null) {
+ project.setExpression(projectInputClone);
+ }
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LORegexp)
+ */
+ @Override
+ protected void visit(LORegexp binOp) throws VisitorException {
+ this.visit((BinaryExpressionOperator)binOp);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOSort)
+ */
+ @Override
+ protected void visit(LOSort s) throws VisitorException {
+ super.visit(s);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOSplit)
+ */
+ @Override
+ protected void visit(LOSplit split) throws VisitorException {
+ List<LogicalOperator> splitOutputs = split.getOutputs();
+ ArrayList<LogicalOperator> splitOutputClones = new ArrayList<LogicalOperator>(splitOutputs.size());
+ for(LogicalOperator splitOutput: splitOutputs) {
+ LogicalOperator splitOutputClone = mOpToCloneMap.get(splitOutput);
+ if(splitOutputClone != null) {
+ splitOutputClones.add(splitOutputClone);
+ } else {
+ splitOutputClones.add(splitOutput);
+ }
+ }
+ split.setOutputs(splitOutputClones);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOSplitOutput)
+ */
+ @Override
+ protected void visit(LOSplitOutput sop) throws VisitorException {
+ super.visit(sop);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOStore)
+ */
+ @Override
+ protected void visit(LOStore store) throws VisitorException {
+ //TODO
+ //LOStore cloning is not implemented
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOStream)
+ */
+ @Override
+ protected void visit(LOStream stream) throws VisitorException {
+ //TODO
+ //LOStream cloning is not implemented
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOSubtract)
+ */
+ @Override
+ public void visit(LOSubtract op) throws VisitorException {
+ this.visit((BinaryExpressionOperator)op);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOUnion)
+ */
+ @Override
+ protected void visit(LOUnion u) throws VisitorException {
+ super.visit(u);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.LOUserFunc)
+ */
+ @Override
+ protected void visit(LOUserFunc func) throws VisitorException {
+ List<ExpressionOperator> args = func.getArguments();
+ List<ExpressionOperator> argsClone = new ArrayList<ExpressionOperator>(args.size());
+ for(ExpressionOperator arg: args) {
+ ExpressionOperator argClone = (ExpressionOperator)mOpToCloneMap.get(arg);
+ if(argClone != null) {
+ argsClone.add(argClone);
+ } else {
+ argsClone.add(arg);
+ }
+ }
+ func.setArguments(argsClone);
+ }
+
+ /**
+ * @see org.apache.pig.impl.logicalLayer.LOVisitor#visit(org.apache.pig.impl.logicalLayer.UnaryExpressionOperator)
+ */
+ @Override
+ protected void visit(UnaryExpressionOperator uniOp) throws VisitorException {
+ ExpressionOperator operandClone = (ExpressionOperator)mOpToCloneMap.get(uniOp.getOperand());
+ if(operandClone != null) {
+ uniOp.setOperand(operandClone);
+ }
+ }
+
+}
Added: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloner.java?rev=714180&view=auto
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloner.java (added)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanCloner.java Fri Nov 14 14:56:50 2008
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.pig.impl.plan.PlanException;
+import org.apache.pig.impl.plan.PlanWalker;
+import org.apache.pig.impl.plan.VisitorException;
+
+/**
+ * LogicalPlanCloner provides the only mechanism of cloning a logical plan and hence the
+ * the logical operators in the plan.
+ */
+public class LogicalPlanCloner {
+
+ private LogicalPlan mPlan;
+
+ /**
+ * @param plan logical plan to be cloned
+ */
+ public LogicalPlanCloner(LogicalPlan plan) {
+ mPlan = plan;
+ }
+
+ public LogicalPlan getClonedPlan() throws CloneNotSupportedException {
+ LogicalPlanCloneHelper lpCloneHelper = new LogicalPlanCloneHelper(mPlan, new HashMap<LogicalOperator, LogicalOperator>());
+ LogicalPlan clonedPlan = lpCloneHelper.getClonedPlan();
+ LogicalPlanCloneHelper.resetState();
+ return clonedPlan;
+ }
+
+}
Added: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/RemoveRedundantOperators.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/RemoveRedundantOperators.java?rev=714180&view=auto
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/RemoveRedundantOperators.java (added)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/RemoveRedundantOperators.java Fri Nov 14 14:56:50 2008
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.util.List;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.Map;
+import java.util.ArrayList;
+
+import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.pig.impl.plan.PlanWalker;
+import org.apache.pig.impl.plan.DepthFirstWalker;
+import org.apache.pig.impl.plan.DependencyOrderWalker;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.plan.PlanException;
+import org.apache.pig.impl.logicalLayer.parser.ParseException;
+import org.apache.pig.impl.plan.OperatorKey;
+import org.apache.pig.impl.util.MultiMap;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.data.DataType;
+
+/**
+ * A visitor to remove redundant operators in a plan
+ */
+public class RemoveRedundantOperators extends
+ LOVisitor {
+
+ public RemoveRedundantOperators(LogicalPlan plan) {
+ super(plan, new DependencyOrderWalker<LogicalOperator, LogicalPlan>(plan));
+ }
+
+ /**
+ *
+ * @param project
+ * the logical project operator that has to be visited
+ * @throws VisitorException
+ */
+ protected void visit(LOProject project) throws VisitorException {
+ LogicalPlan currentPlan = (LogicalPlan)mCurrentWalker.getPlan();
+
+ //if the project is a project(*) and if there are predecessors and successors that are
+ //1. both relational operators OR
+ //2. both expression operators
+ //then the project(*) can be removed and the input and outputs short circuited, i.e. directly connected
+ if(project.isStar()) {
+
+ List<LogicalOperator> projectSuccessors = currentPlan.getSuccessors(project);
+ List<LogicalOperator> projectPredecessors = currentPlan.getPredecessors(project);
+
+ if(((projectSuccessors != null) && (projectSuccessors.size() > 0))
+ && ((projectPredecessors != null) && (projectPredecessors.size() > 0))) {
+
+ //Making copies to avoid concurrent modification exceptions
+ List<LogicalOperator> successors = new ArrayList(currentPlan.getSuccessors(project));
+ List<LogicalOperator> predecessors = new ArrayList(currentPlan.getPredecessors(project));
+
+ //if the project(*) cannot be removed
+ boolean removeProject = true;
+
+ for(LogicalOperator projectPred: predecessors) {
+ for(LogicalOperator projectSucc: successors) {
+ if (((projectPred instanceof ExpressionOperator) && (projectSucc instanceof ExpressionOperator))
+ || (!(projectPred instanceof ExpressionOperator) && !(projectSucc instanceof ExpressionOperator))) {
+ try {
+ currentPlan.disconnect(projectPred, project);
+ currentPlan.disconnect(project, projectSucc);
+ currentPlan.connect(projectPred, projectSucc);
+ patchInputReference(projectSucc, project, projectPred);
+ } catch (PlanException pe) {
+ throw new VisitorException(pe.getMessage(), pe);
+ }
+ } else {
+ removeProject = false;
+ }
+ }
+ }
+ if(removeProject) {
+ currentPlan.remove(project);
+ }
+ }
+ }
+ }
+
+ private void patchInputReference(LogicalOperator op, LogicalOperator prevInput, LogicalOperator newInput) {
+ //TODO
+ //Using reference comparison here as operators do not have equals() method yet
+ //Depending on the successor of prevInput, fix the referenes to point to newInput
+ if(op instanceof BinaryExpressionOperator) {
+ BinaryExpressionOperator binOp = (BinaryExpressionOperator)op;
+ if(prevInput == binOp.getLhsOperand()) {
+ binOp.setLhsOperand((ExpressionOperator)newInput);
+ } else if(prevInput == binOp.getRhsOperand()) {
+ binOp.setRhsOperand((ExpressionOperator)newInput);
+ }
+ } else if (op instanceof UnaryExpressionOperator) {
+ UnaryExpressionOperator uniOp = (UnaryExpressionOperator)op;
+ if(prevInput == uniOp.getOperand()) {
+ uniOp.setOperand((ExpressionOperator)newInput);
+ }
+ } else if (op instanceof LOBinCond) {
+ LOBinCond binCond = (LOBinCond)op;
+ if(prevInput == binCond.getLhsOp()) {
+ binCond.setLhsOp((ExpressionOperator)newInput);
+ } else if(prevInput == binCond.getRhsOp()) {
+ binCond.setRhsOp((ExpressionOperator)newInput);
+ } else if(prevInput == binCond.getCond()) {
+ binCond.setCond((ExpressionOperator)newInput);
+ }
+ } else if (op instanceof LOCast) {
+ LOCast cast = (LOCast)op;
+ if(prevInput == cast.getExpression()) {
+ cast.setExpression((ExpressionOperator)newInput);
+ }
+ } else if (op instanceof LOMapLookup) {
+ LOMapLookup map = (LOMapLookup)op;
+ if(prevInput == map.getMap()) {
+ map.setMap((ExpressionOperator)newInput);
+ }
+ } else if (op instanceof LOUserFunc) {
+ LOUserFunc userFunc = (LOUserFunc)op;
+ List<ExpressionOperator> args = userFunc.getArguments();
+ ArrayList<ExpressionOperator> newArgs = new ArrayList<ExpressionOperator>(args.size());
+ for(ExpressionOperator expOp: args) {
+ if(prevInput == expOp) {
+ newArgs.add((ExpressionOperator)newInput);
+ } else {
+ newArgs.add(expOp);
+ }
+ }
+ userFunc.setArguments(newArgs);
+ } else if (op instanceof LOProject) {
+ LOProject proj = (LOProject)op;
+ if(prevInput == proj.getExpression()) {
+ proj.setExpression(newInput);
+ }
+ }
+ }
+
+}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java Fri Nov 14 14:56:50 2008
@@ -70,6 +70,10 @@
return mOperand;
}
+ public void setOperand(ExpressionOperator eOp) {
+ mOperand = eOp;
+ }
+
@Override
public void visit(LOVisitor v) throws VisitorException {
v.visit(this);
@@ -80,4 +84,15 @@
return false;
}
+ /**
+ * @see org.apache.pig.impl.logicalLayer.ExpressionOperator#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ UnaryExpressionOperator unExOpClone = (UnaryExpressionOperator)super.clone();
+ return unExOpClone;
+ }
+
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Fri Nov 14 14:56:50 2008
@@ -55,7 +55,7 @@
import org.apache.pig.data.DataBag;
import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
-import org.apache.pig.impl.logicalLayer.ProjectStarTranslator;
+import org.apache.pig.impl.plan.VisitorException;
public class QueryParser {
@@ -1699,7 +1699,24 @@
}
}
}
- foreachPlans.add(generatePlan);
+ LogicalPlanCloner lpCloner = new LogicalPlanCloner(generatePlan);
+ LogicalPlan generatePlanClone;
+ try {
+ generatePlanClone = lpCloner.getClonedPlan();
+ } catch (CloneNotSupportedException cnse) {
+ log.error(cnse.getMessage());
+ cnse.printStackTrace();
+ ParseException pe = new ParseException("Not able to clone foreach plan");
+ pe.initCause(cnse);
+ throw pe;
+ }
+ RemoveRedundantOperators removeOperators = new RemoveRedundantOperators(generatePlanClone);
+ try {
+ removeOperators.visit();
+ } catch (VisitorException ve) {
+ throw new ParseException("Could not remove redundant operators in foreach plan.").initCause(ve);
+ }
+ foreachPlans.add(generatePlanClone);
}
resetGenerateState();
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java Fri Nov 14 14:56:50 2008
@@ -1161,6 +1161,7 @@
OperatorKey newKey = genNewOperatorKey(uniOp) ;
LOCast cast = new LOCast(currentPlan, newKey, input, toType) ;
+ currentPlan.add(cast);
currentPlan.disconnect(input, uniOp) ;
try {
currentPlan.connect(input, cast) ;
@@ -1172,6 +1173,7 @@
throw err ;
}
+ uniOp.setOperand(cast);
this.visit(cast);
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/plan/Operator.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/plan/Operator.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/plan/Operator.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/plan/Operator.java Fri Nov 14 14:56:50 2008
@@ -28,7 +28,7 @@
/**
* Base class for all types of operators.
*/
-abstract public class Operator<V extends PlanVisitor> implements Serializable, Comparable<Operator> {
+abstract public class Operator<V extends PlanVisitor> implements Serializable, Comparable<Operator>, Cloneable {
private static final long serialVersionUID = 1L;
/**
@@ -115,7 +115,17 @@
return mKey.compareTo(o.mKey);
}
-
-
+ /**
+ * @see java.lang.Object#clone()
+ * Do not use the clone method directly. Operators are cloned when logical plans
+ * are cloned using {@link LogicalPlanCloner}
+ */
+ @Override
+ protected Object clone() throws CloneNotSupportedException {
+ Object o = super.clone();
+ Operator opClone = (Operator)o;
+ opClone.mKey = new OperatorKey(mKey.scope, NodeIdGenerator.getGenerator().getNextNodeId(mKey.scope));
+ return opClone;
+ }
}
Modified: hadoop/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java (original)
+++ hadoop/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java Fri Nov 14 14:56:50 2008
@@ -48,7 +48,7 @@
* for graph manipulators (such as the validators and optimizers) to
* understand the internals of various nodes.
*/
-public abstract class OperatorPlan<E extends Operator> implements Iterable, Serializable {
+public abstract class OperatorPlan<E extends Operator> implements Iterable, Serializable, Cloneable {
protected Map<E, OperatorKey> mOps;
protected Map<OperatorKey, E> mKeys;
protected MultiMap<E, E> mFromEdges;
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java Fri Nov 14 14:56:50 2008
@@ -833,4 +833,139 @@
}
+ @Test
+ public void testNestedPlanForCloning() throws Exception{
+ int LOOP_COUNT = 10;
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ Random r = new Random();
+ for(int i = 0; i < LOOP_COUNT; i++) {
+ for(int j=0;j<LOOP_COUNT;j+=2){
+ ps.println(i+"\t"+j);
+ ps.println(i+"\t"+j);
+ }
+ }
+ ps.close();
+
+ String tmpOutputFile = FileLocalizer.getTemporaryPath(null,
+ pigServer.getPigContext()).toString();
+ pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pigServer.registerQuery("B = group A by $0;");
+ String query = "C = foreach B {"
+ + "C1 = filter A by not($0 <= -1);"
+ + "C2 = distinct C1;"
+ + "C3 = distinct A;"
+ + "C4 = order A by $0;"
+ + "generate (group + 1) * 10, COUNT(C4), COUNT(C2), SUM(C2.$1)," + TitleNGrams.class.getName() + "(C3), MAX(C3.$1), C2;"
+ + "};";
+
+ pigServer.registerQuery(query);
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+ if(!iter.hasNext()) fail("No output found");
+ int numIdentity = 0;
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ assertEquals((Integer)((numIdentity + 1) * 10), (Integer)t.get(0));
+ assertEquals((Long)10L, (Long)t.get(1));
+ assertEquals((Long)5L, (Long)t.get(2));
+ assertEquals(LOOP_COUNT*2.0, (Double)t.get(3), 0.01);
+ assertEquals(8.0, (Double)t.get(5), 0.01);
+ assertEquals(5L, ((DataBag)t.get(6)).size());
+ assertEquals(7, t.size());
+ ++numIdentity;
+ }
+ assertEquals(LOOP_COUNT, numIdentity);
+ }
+
+ @Test
+ public void testArithmeticCloning() throws Exception{
+ int LOOP_COUNT = 10;
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ Random r = new Random();
+ for(int i = 0; i < LOOP_COUNT; i++) {
+ for(int j=0;j<LOOP_COUNT;j+=2){
+ ps.println(i+"\t"+j);
+ ps.println(i+"\t"+j);
+ }
+ }
+ ps.close();
+
+ String tmpOutputFile = FileLocalizer.getTemporaryPath(null,
+ pigServer.getPigContext()).toString();
+ pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pigServer.registerQuery("B = distinct A;");
+ String query = "C = foreach B {"
+ + "C1 = $1 - $0;"
+ + "C2 = $1%2;"
+ + "C3 = ($1 == 0? 0 : $0/$1);"
+ + "generate C1, C2, C3;"
+ + "};";
+
+ pigServer.registerQuery(query);
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+ if(!iter.hasNext()) fail("No output found");
+
+ int numRows = 0;
+ for(int i = 0; i < LOOP_COUNT; i++) {
+ for(int j = 0; j < LOOP_COUNT; j+=2){
+ Tuple t = null;
+ if(iter.hasNext()) t = iter.next();
+ assertEquals(3, t.size());
+ assertEquals(new Double(j - i), (Double)t.get(0), 0.01);
+ assertEquals((Integer)(j%2), (Integer)t.get(1));
+ if(j == 0) {
+ assertEquals(0.0, (Double)t.get(2), 0.01);
+ } else {
+ assertEquals((Double)((double)i/j), (Double)t.get(2), 0.01);
+ }
+ ++numRows;
+ }
+ }
+
+ assertEquals((LOOP_COUNT * LOOP_COUNT)/2, numRows);
+ }
+
+ @Test
+ public void testExpressionReUse() throws Exception{
+ int LOOP_COUNT = 10;
+ File tmpFile = File.createTempFile("test", "txt");
+ PrintStream ps = new PrintStream(new FileOutputStream(tmpFile));
+ Random r = new Random();
+ for(int i = 0; i < LOOP_COUNT; i++) {
+ for(int j=0;j<LOOP_COUNT;j+=2){
+ ps.println(i+"\t"+j);
+ ps.println(i+"\t"+j);
+ }
+ }
+ ps.close();
+
+ String tmpOutputFile = FileLocalizer.getTemporaryPath(null,
+ pigServer.getPigContext()).toString();
+ pigServer.registerQuery("A = LOAD '" + Util.generateURI(tmpFile.toString()) + "';");
+ pigServer.registerQuery("B = distinct A;");
+ String query = "C = foreach B {"
+ + "C1 = $0 + $1;"
+ + "C2 = C1 + $0;"
+ + "generate C1, C2;"
+ + "};";
+
+ pigServer.registerQuery(query);
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+ if(!iter.hasNext()) fail("No output found");
+
+ int numRows = 0;
+ for(int i = 0; i < LOOP_COUNT; i++) {
+ for(int j = 0; j < LOOP_COUNT; j+=2){
+ Tuple t = null;
+ if(iter.hasNext()) t = iter.next();
+ assertEquals(2, t.size());
+ assertEquals(new Double(i + j), (Double)t.get(0), 0.01);
+ assertEquals(new Double(i + j + i), (Double)t.get(1));
+ ++numRows;
+ }
+ }
+
+ assertEquals((LOOP_COUNT * LOOP_COUNT)/2, numRows);
+ }
}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java Fri Nov 14 14:56:50 2008
@@ -111,6 +111,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -128,7 +129,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
//System.out.println(compiledPlan.compareTo(goldenPlan)==0);
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
public void testSort() throws VisitorException, IOException {
@@ -139,6 +140,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -158,7 +160,7 @@
System.out.println("-------------");
//System.out.println(compiledPlan.compareTo(goldenPlan)==0);
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
public void testDistinct() throws VisitorException, IOException {
@@ -169,6 +171,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -187,7 +190,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
//System.out.println(compiledPlan.compareTo(goldenPlan)==0);
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
public void testCogroup() throws VisitorException, IOException {
@@ -200,6 +203,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -235,6 +239,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -253,7 +258,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
//System.out.println(compiledPlan.compareTo(goldenPlan)==0);
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
public void testComparison() throws VisitorException, IOException {
@@ -267,6 +272,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -285,7 +291,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
//System.out.println(compiledPlan.compareTo(goldenPlan)==0);
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
@Test
@@ -299,6 +305,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -317,7 +324,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
//System.out.println(compiledPlan.compareTo(goldenPlan)==0);
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
@@ -332,6 +339,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -350,7 +358,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
@Test
@@ -362,6 +370,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -380,7 +389,7 @@
System.out.println(compiledPlan);
System.out.println("-------------");
- assertEquals(true, compiledPlan.compareTo(goldenPlan) == 0);
+ assertEquals(compiledPlan, goldenPlan);
}
@Test
@@ -393,6 +402,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -415,8 +425,7 @@
System.out.println();
System.out.println(compiledPlan);
System.out.println("-------------");
- boolean flag = false;
- //System.out.println("GoldenPlan1\n" + goldenPlan1 + "\nGoldenPlan2\n" + goldenPlan2);
+
if(compiledPlan.compareTo(goldenPlan1) == 0 || compiledPlan.compareTo(goldenPlan2) == 0) {
// good
}
@@ -446,6 +455,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
@@ -454,7 +464,6 @@
return;
}
-// compiledPlan += "\n"; //for the string compare, the files contain an additional \n
FileInputStream fis1 = new FileInputStream("test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld");
FileInputStream fis2 = new FileInputStream("test/org/apache/pig/test/data/GoldenFiles/IsNull2.gld");
byte[] b1 = new byte[MAX_SIZE];
@@ -469,8 +478,7 @@
System.out.println();
System.out.println(compiledPlan);
System.out.println("-------------");
- boolean flag = false;
- //System.out.println("GoldenPlan1\n" + goldenPlan1 + "\nGoldenPlan2\n" + goldenPlan2);
+
if(compiledPlan.compareTo(goldenPlan1) == 0 || compiledPlan.compareTo(goldenPlan2) == 0) {
// good
}
@@ -496,6 +504,7 @@
int MAX_SIZE = 100000;
ByteArrayOutputStream baos = new ByteArrayOutputStream();
pp.explain(baos);
+ baos.write((int)'\n');
String compiledPlan = baos.toString();
if(generate){
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalOptimizer.java Fri Nov 14 14:56:50 2008
@@ -67,7 +67,7 @@
String actualPlan = printLimitGraph(plan);
System.out.println("We get:");
System.out.println(actualPlan);
- assertEquals(goldenPlan, actualPlan);
+ assertEquals(goldenPlan, actualPlan + "\n");
}
@Test
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java Fri Nov 14 14:56:50 2008
@@ -1698,6 +1698,24 @@
}
+ @Test
+ public void testQuery113() throws FrontendException, ParseException {
+ LogicalPlan lp;
+ LOForEach foreach;
+ LOSort sort;
+
+ buildPlan("a = load 'one' as (name, age, gpa);");
+
+ lp = buildPlan("b = foreach a {exp1 = age + gpa; exp2 = exp1 + age; generate exp1, exp2;};");
+ foreach = (LOForEach) lp.getLeaves().get(0);
+
+ for(LogicalPlan foreachPlan: foreach.getForEachPlans()) {
+ printPlan(foreachPlan);
+ assertTrue(checkPlanForProjectStar(foreachPlan) == false);
+ }
+
+ }
+
private Schema getSchemaFromString(String schemaString) throws ParseException {
return getSchemaFromString(schemaString, DataType.BYTEARRAY);
}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan1.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan1.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan1.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan1.dot Fri Nov 14 14:56:50 2008
@@ -1,4 +1,4 @@
digraph graph1 {
LOLoad18 -> LOSort20;
LOSort20 [limit="100"];
-}
\ No newline at end of file
+}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan2.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan2.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan2.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan2.dot Fri Nov 14 14:56:50 2008
@@ -1,4 +1,4 @@
digraph graph1 {
LOLoad22 -> LOLimit23;
LOLimit23 [limit="10"];
-}
\ No newline at end of file
+}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan3.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan3.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan3.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan3.dot Fri Nov 14 14:56:50 2008
@@ -7,4 +7,4 @@
LOLimit30 -> LOCross27;
LOCross27 -> LOLimit28;
LOLimit28 [limit="100"];
-}
\ No newline at end of file
+}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot Fri Nov 14 14:56:50 2008
@@ -1,7 +1,7 @@
digraph graph1 {
LOLoad31 -> LOCogroup33;
- LOCogroup33 -> LOForEach36;
- LOForEach36 [hasflat="true"];
- LOForEach36 -> LOLimit37;
- LOLimit37 [limit="100"];
-}
\ No newline at end of file
+ LOCogroup33 -> LOForEach37;
+ LOForEach37 [hasflat="true"];
+ LOForEach37 -> LOLimit38;
+ LOLimit38 [limit="100"];
+}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot Fri Nov 14 14:56:50 2008
@@ -1,6 +1,6 @@
digraph graph1 {
- LOLoad38 -> LOLimit42;
- LOLimit42 [limit="100"];
- LOLimit42 -> LOForEach41;
- LOForEach41 [hasflat="false"];
-}
\ No newline at end of file
+ LOLoad39 -> LOLimit44;
+ LOLimit44 [limit="100"];
+ LOLimit44 -> LOForEach43;
+ LOForEach43 [hasflat="false"];
+}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot Fri Nov 14 14:56:50 2008
@@ -1,4 +1,4 @@
digraph graph1 {
- LOLoad43 -> LOLimit44;
- LOLimit44 [limit="20"];
-}
\ No newline at end of file
+ LOLoad45 -> LOLimit46;
+ LOLimit46 [limit="20"];
+}
Modified: hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot?rev=714180&r1=714179&r2=714180&view=diff
==============================================================================
--- hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot (original)
+++ hadoop/pig/branches/types/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot Fri Nov 14 14:56:50 2008
@@ -1,6 +1,6 @@
digraph graph1 {
- LOLoad47 -> LOForEach50;
- LOForEach50 [hasflat="true"];
- LOForEach50 -> LOLimit51;
- LOLimit51 [limit="100"];
-}
\ No newline at end of file
+ LOLoad49 -> LOForEach53;
+ LOForEach53 [hasflat="true"];
+ LOForEach53 -> LOLimit54;
+ LOLimit54 [limit="100"];
+}