You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/08 23:25:54 UTC

svn commit: r654629 [1/4] - in /incubator/pig/branches/types: ./ src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/logicalLayer/ src/org/apache/pig/impl/logicalLayer/parser/ src/org/apache/pig/impl/logicalLayer/schema/ src/org/apache/pig/impl/logic...

Author: gates
Date: Thu May  8 14:25:22 2008
New Revision: 654629

URL: http://svn.apache.org/viewvc?rev=654629&view=rev
Log:
Santhosh's massive parser rewrite.


Added:
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
Removed:
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/BinCondSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/CompositeEvalSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/ConstSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/EvalSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/EvalSpecPrinter.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/EvalSpecVisitor.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/FilterSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/FuncEvalSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/GenerateSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/MapLookupSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/ProjectSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/SimpleEvalSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/SortDistinctSpec.java
    incubator/pig/branches/types/src/org/apache/pig/impl/eval/StarSpec.java
Modified:
    incubator/pig/branches/types/build.xml
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSubtract.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
    incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
    incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
    incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java

Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Thu May  8 14:25:22 2008
@@ -144,7 +144,7 @@
                  **/test/FakeFSOutputStream.java, **/test/TestPackage.java, **/test/TestForEach.java,
         		**/test/TestLocalRearrange.java, **/test/TestPOUserFunc.java,
         		**/test/TestPODistinct.java, **/test/TestPOSort.java,
-        		**/test/TestSchema.java,
+        		**/test/TestSchema.java, **/test/TestLogicalPlanBuilder.java,
                 **/test/FakeFSInputStream.java, **/test/Util.java,
                 **/logicalLayer/*.java, **/logicalLayer/parser/NodeIdGenerator.java,
                 **/logicalLayer/schema/*.java, **/physicalLayer/topLevelOperators/*.java,
@@ -265,6 +265,7 @@
                 	<include name="**/TestInputOutputFileValidator.java" /> 
                 	<include name="**/TestTypeCheckingValidator.java" /> 
                 	<include name="**/TestSchema.java" /> 
+                	<include name="**/TestLogicalPlanBuilder.java" /> 
                     <!--
                     <include name="**/*Test*.java" />
                     <exclude name="**/TestLargeFile.java" />

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java Thu May  8 14:25:22 2008
@@ -20,6 +20,8 @@
 
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 /**
  * This abstract class represents the logical Binary Expression Operator
@@ -32,6 +34,7 @@
     private static final long serialVersionUID = 2L;
     private ExpressionOperator mLhsOperand; //left hand side operand
     private ExpressionOperator mRhsOperand; //right hand side operand
+    private static Log log = LogFactory.getLog(BinaryExpressionOperator.class);
 
     /**
      * @param plan
@@ -53,6 +56,23 @@
         mRhsOperand = rhsOperand;
     }
 
+    /**
+     * @param plan
+     *            Logical plan this operator is a part of.
+     * @param k
+     *            Operator key to assign to this node.
+     * @param lhsOperand
+     *            ExpressionOperator the left hand side operand
+     * @param rhsOperand
+     *            ExpressionOperator the right hand side operand
+     */
+    public BinaryExpressionOperator(LogicalPlan plan, OperatorKey k,
+            ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
+        super(plan, k);
+        mLhsOperand = lhsOperand;
+        mRhsOperand = rhsOperand;
+    }
+    
     public ExpressionOperator getLhsOperand() {
         return mLhsOperand;
     }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java Thu May  8 14:25:22 2008
@@ -19,10 +19,13 @@
 
 import java.util.List;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public abstract class ExpressionOperator extends LogicalOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(ExpressionOperator.class);
 
     /**
      * @param plan
@@ -37,6 +40,17 @@
         super(plan, k, rp);
     }
 
+    /**
+     * @param plan
+     *            Logical plan this operator is a part of.
+     * @param k
+     *            Operator key to assign to this node.
+     */
+    public ExpressionOperator(LogicalPlan plan, OperatorKey k) {
+        super(plan, k);
+    }
+
+    
     @Override
     public boolean supportsMultipleOutputs() {
         return false;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOAdd extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOAdd.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOAdd(LogicalPlan plan, OperatorKey k, int rp,
+    public LOAdd(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOAnd extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOAnd.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOAnd(LogicalPlan plan, OperatorKey k, int rp,
+    public LOAnd(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java Thu May  8 14:25:22 2008
@@ -41,9 +41,6 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param cond
      *            ExpressionOperator the expression specifying condition
      * @param lhsOp
@@ -52,10 +49,10 @@
      *            ExpressionOperator query to be executed when condition is
      *            false
      */
-    public LOBinCond(LogicalPlan plan, OperatorKey k, int rp,
+    public LOBinCond(LogicalPlan plan, OperatorKey k,
             ExpressionOperator cond, ExpressionOperator lhsOp,
             ExpressionOperator rhsOp) {
-        super(plan, k, rp);
+        super(plan, k);
         mCond = cond;
         mLhsOp = lhsOp;
         mRhsOp = rhsOp;
@@ -89,9 +86,12 @@
 
     @Override
     public Schema getSchema() throws FrontendException {
+		//TODO
+		//We need a check of LHS and RHS schemas
         if (!mIsSchemaComputed && (null == mSchema)) {
             try {
                 mSchema = mLhsOp.getSchema();
+                //mSchema = mLhsPlan.getRoots().get(0).getSchema();
                 mIsSchemaComputed = true;
             } catch (FrontendException ioe) {
                 mSchema = null;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java Thu May  8 14:25:22 2008
@@ -35,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param expr
      *            the expression whose type has to be cast
      * @param type
      *            the type to which the expression is cast
      */
-    public LOCast(LogicalPlan plan, OperatorKey k, int rp,
+    public LOCast(LogicalPlan plan, OperatorKey k,
             ExpressionOperator expr, byte type) {
-        super(plan, k, rp);
+        super(plan, k);
         mExpr = expr;
         mType = type;
     }// End Constructor LOCast

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java Thu May  8 14:25:22 2008
@@ -18,20 +18,35 @@
 package org.apache.pig.impl.logicalLayer;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.HashMap;
 
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.impl.plan.MultiMap;
 
 public class LOCogroup extends LogicalOperator {
     private static final long serialVersionUID = 2L;
 
-    private ArrayList<String> mInputs;
-    private ArrayList<ExpressionOperator> mGroupByCols;
+    /**
+     * Cogroup contains a list of logical operators corresponding to the
+     * relational operators and a list of generates for each relational
+     * operator. Each generate operator in turn contains a list of expressions
+     * for the columns that are projected
+     */
+    private ArrayList<LogicalOperator> mInputs;
+    private boolean[] mIsInner;
+    private static Log log = LogFactory.getLog(LOCogroup.class);
+    private MultiMap<LogicalOperator, LogicalPlan> mGroupByPlans;
 
     /**
      * 
@@ -39,24 +54,28 @@
      *            LogicalPlan this operator is a part of.
      * @param key
      *            OperatorKey for this operator
-     * @param rp
-     *            Requested level of parallelism to be used
      * @param groupByCols
      *            the group by columns
      */
-    public LOCogroup(LogicalPlan plan, OperatorKey k, int rp,
-            ArrayList<String> inputs, ArrayList<ExpressionOperator> groupByCols) {
-        super(plan, k, rp);
+    public LOCogroup(LogicalPlan plan, OperatorKey k,
+            ArrayList<LogicalOperator> inputs,
+            MultiMap<LogicalOperator, LogicalPlan> groupByPlans, boolean[] isInner) {
+        super(plan, k);
         mInputs = inputs;
-        mGroupByCols = groupByCols;
+        mGroupByPlans = groupByPlans;
+        mIsInner = isInner;
     }
 
-    public List<String> getInputs() {
+    public List<LogicalOperator> getInputs() {
         return mInputs;
     }
 
-    public List<ExpressionOperator> getGroupByCols() {
-        return mGroupByCols;
+    public MultiMap<LogicalOperator, LogicalPlan> getGroupByPlans() {
+        return mGroupByPlans;
+    }    
+
+    public boolean[] getInner() {
+        return mIsInner;
     }
 
     @Override
@@ -77,7 +96,7 @@
     @Override
     public Schema getSchema() throws FrontendException {
         // TODO create schema
-        /**
+        /*
          * Dumping my understanding of how the schema of a Group/CoGroup will
          * look. The first field of the resulting tuple will have the alias
          * 'group'. The schema for this field is a union of the group by columns
@@ -85,19 +104,150 @@
          * the alias of the input as the alias for a bag that contains the
          * tuples from the input that match the grouping criterion
          */
-        if (null == mSchema) {
+        if (!mIsSchemaComputed && (null == mSchema)) {
             List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(
-                    mGroupByCols.size() + 1);
+                    mInputs.size() + 1);
             // one more to account for the "group"
             // the alias of the first field is group and hence the
             // string "group"
             // TODO The type of the field named "group" requires
             // type promotion and the like
-            fss.add(new Schema.FieldSchema("group", null));
-            for (String input : mInputs) {
-                fss.add(new Schema.FieldSchema(input, DataType.BAG));
+
+            /*
+             * Here goes an attempt to describe how the schema for the first
+             * column - 'group' should look like If the number of group by
+             * columns = 1 then the schema for 'group' is the
+             * schema(fieldschema(col)) If the number of group by columns > 1
+             * then find the set union of the group by columns and form the
+             * schema as schema(list<fieldschema of the cols>)
+			 * The parser will ensure that the number of group by columns are
+			 * the same across all inputs. The computation of the schema for group
+			 * is 
+             */
+
+            Schema groupBySchema = null;
+            List<Schema.FieldSchema> groupByFss = new ArrayList<Schema.FieldSchema>();
+            Set<String> groupByAliases = new HashSet<String>();
+            Map<String, Boolean> lookup = new HashMap<String, Boolean>();
+            MultiMap<String, ExpressionOperator> aliasExop = new MultiMap<String, ExpressionOperator>();
+            MultiMap<Integer, String> positionAlias= new MultiMap<Integer, String>();
+            
+            for (LogicalOperator op : mInputs) {
+                log.debug("GBY Input: " + op.getClass().getName());
+				for(LogicalPlan plan: mGroupByPlans.get(op)) {
+	                int position = 0;
+	                for(LogicalOperator eOp: plan.getRoots()) {
+	                    Schema eOpSchema = eOp.getSchema();
+	                    log.debug("Computing the lookup tables");
+	                    if (null != eOpSchema) {
+	                        for (String alias : eOpSchema.getAliases()) {
+	                            log.debug("Adding alias to GBY: " + alias);
+	                            groupByAliases.add(alias);
+	                            lookup.put(alias, false);
+	                            aliasExop.put(alias, (ExpressionOperator)eOp);                            
+	                            positionAlias.put(position, alias);
+	                        }
+	                    }
+	                }
+	                ++position;
+				}
+            }
+            
+            log.debug("Computed the lookup table");
+
+            int arity = mGroupByPlans.get(mInputs.get(0)).size();
+            log.debug("Arity: " + arity);
+            for (int i = 0; i < arity; ++i) {
+                Collection<String> cAliases;
+                cAliases = positionAlias.get(i);
+                if(null != cAliases) {
+                    Object[] aliases = cAliases.toArray();
+                    for(int j = 0; j < aliases.length; ++j) {
+                        String alias = (String) aliases[j];
+                        if(null != alias) {
+                            Collection<ExpressionOperator> cEops = aliasExop.get(alias);
+                            if(null != cEops) {
+                                ExpressionOperator eOp = (ExpressionOperator) (cEops.toArray())[0];
+                                if(null != eOp) {
+                                    if(!lookup.get(alias)) {
+                                        groupByFss.add(new Schema.FieldSchema(alias, eOp.getSchema()));
+                                        lookup.put(alias, true);
+                                    } else {
+                                        if(j < aliases.length) {
+                                            continue;
+                                        } else {
+                                            //we have seen this alias before
+                                            //just add the schema of the expression operator with the null alias
+                                            groupByFss.add(new Schema.FieldSchema(null, eOp.getSchema()));
+                                            break;
+                                        }
+                                    }
+                                } else {
+                                    //should not be here
+                                    log.debug("Cannot be here: we cannot have a collection of null expression operators");
+                                }
+                            } else {
+                                //should not be here
+                                log.debug("Cannot be here: we cannot have an alias without an expression operator");
+                            }
+                        } else {
+                            //should not be here
+                            log.debug("Cannot be here: we cannot have a collection of null aliases ");
+                        }
+                    }
+                } else {
+                    //We do not have any alias for this position in the group by columns
+                    //We have positions $1, $2, etc.
+                    //The schema for these columns is the schema of the expression operatore
+                    //and so the alias is null
+                    groupByFss.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));                    
+                }
+            }            
+
+            groupBySchema = new Schema(groupByFss);
+            log.debug("Printing group by schema aliases");
+            groupBySchema.printAliases();
+
+			if(1 == arity) {
+				log.debug("Arity == 1");
+				byte groupByType = groupByFss.get(0).type;
+				Schema groupSchema = groupByFss.get(0).schema;
+				log.debug("Type == " + DataType.findTypeName(groupByType));
+				if(DataType.TUPLE == groupByType || DataType.BAG == groupByType) { 
+					if(null != groupSchema) {
+						Schema innerSchema = groupSchema.getFields().get(0).schema;
+						fss.add(new Schema.FieldSchema("group", innerSchema, groupByType));
+					} else {
+						fss.add(new Schema.FieldSchema("group", groupSchema, groupByType));
+					}
+					log.debug("Printing the aliases of the single group by column");
+					groupSchema.printAliases();
+				} else {
+					fss.add(new Schema.FieldSchema("group", groupByType));
+				}
+			} else {
+            	fss.add(new Schema.FieldSchema("group", groupBySchema));
+			}
+            for (LogicalOperator op : mInputs) {
+                log.debug("Op: " + op.getClass().getName());
+                log.debug("Op Alias: " + op.getAlias());
+                try {
+                    Schema cSchema = op.getSchema();
+                    if (null != cSchema) {
+                        log.debug("Printing constituent schema aliases");
+                        cSchema.printAliases();
+                    }
+                    fss.add(new Schema.FieldSchema(op.getAlias(), op
+                            .getSchema(), DataType.BAG));
+                } catch (FrontendException ioe) {
+                    log.debug("Caught an exception: " + ioe.getMessage());
+                    mIsSchemaComputed = false;
+                    mSchema = null;
+                    throw ioe;
+                }
             }
             mIsSchemaComputed = true;
+            mSchema = new Schema(fss);
         }
         return mSchema;
     }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java Thu May  8 14:25:22 2008
@@ -35,14 +35,11 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param value
      *            the value of the constant
      */
-    public LOConst(LogicalPlan plan, OperatorKey k, int rp, Object value) {
-        super(plan, k, rp);
+    public LOConst(LogicalPlan plan, OperatorKey k, Object value) {
+        super(plan, k);
         mValue = value;
     }// End Constructor LOConst
 

Added: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java?rev=654629&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java Thu May  8 14:25:22 2008
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.io.IOException;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class LOCross extends LogicalOperator {
+
+    private static final long serialVersionUID = 2L;
+    private ArrayList<LogicalOperator> mInputs;
+	private static Log log = LogFactory.getLog(LOCross.class);
+
+    /**
+     * 
+     * @param plan
+     *            Logical plan this operator is a part of.
+     * @param k
+     *            Operator key to assign to this node.
+     */
+    public LOCross(LogicalPlan plan, OperatorKey k, ArrayList<LogicalOperator> inputs) {
+
+        super(plan, k);
+        mInputs = inputs;
+    }
+
+    public List<LogicalOperator>  getInputs() {
+        return mInputs;
+    }
+    
+    public void addInput(LogicalOperator input) {
+        mInputs.add(input);
+    }
+    
+    @Override
+    public Schema getSchema() throws FrontendException {
+        if (!mIsSchemaComputed && (null == mSchema)) {
+            // Get the schema of the parents
+            Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+            List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(s
+                    .size());
+            for (LogicalOperator op : s) {
+                String opAlias = op.getAlias();
+                if (op.getType() == DataType.TUPLE) {
+                    try {
+                        fss.add(new Schema.FieldSchema(opAlias, op
+                                        .getSchema()));
+                    } catch (FrontendException ioe) {
+                        mSchema = null;
+                        mIsSchemaComputed = false;
+                        throw ioe;
+                    }
+                } else {
+                    fss.add(new Schema.FieldSchema(opAlias, op.getType()));
+                }
+            }
+            mSchema = new Schema(fss);
+            mIsSchemaComputed = true;
+        }
+        return mSchema;
+    }
+
+    @Override
+    public String name() {
+        return "Cross " + mKey.scope + "-" + mKey.id;
+    }
+
+    @Override
+    public boolean supportsMultipleInputs() {
+        return true;
+    }
+
+    @Override
+    public boolean supportsMultipleOutputs() {
+        return false;
+    }
+
+    @Override
+    public void visit(LOVisitor v) throws VisitorException {
+        v.visit(this);
+    }
+
+}

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java Thu May  8 14:25:22 2008
@@ -25,10 +25,14 @@
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LODistinct extends LogicalOperator {
 
     private static final long serialVersionUID = 2L;
+    private LogicalOperator mInput;
+    private static Log log = LogFactory.getLog(LODistinct.class);
 
     /**
      * 
@@ -36,23 +40,28 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
+     * @param input
+     *            input over which distinct will be applied
      */
-    public LODistinct(LogicalPlan plan, OperatorKey k, int rp) {
+    public LODistinct(LogicalPlan plan, OperatorKey k, LogicalOperator input) {
 
-        super(plan, k, rp);
+        super(plan, k);
+        mInput = input;
+    }
+
+    public LogicalOperator getInput() {
+        return mInput;
     }
 
     @Override
     public Schema getSchema() throws FrontendException {
         if (!mIsSchemaComputed && (null == mSchema)) {
             // Get the schema of the parent
-            Collection<LogicalOperator> s = mPlan.getSuccessors(this);
+            Collection<LogicalOperator> s = mPlan.getPredecessors(this);
             try {
                 LogicalOperator op = s.iterator().next();
-                if(null == op) {
+                if (null == op) {
+                    log.info("getSchema: Operator not in plan");
                     throw new FrontendException("Could not find operator in plan");
                 }
                 mSchema = s.iterator().next().getSchema();

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LODivide extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LODivide.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LODivide(LogicalPlan plan, OperatorKey k, int rp,
+    public LODivide(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOEqual extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOEqual.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOEqual(LogicalPlan plan, OperatorKey k, int rp,
+    public LOEqual(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java Thu May  8 14:25:22 2008
@@ -22,12 +22,15 @@
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOFilter extends LogicalOperator {
 
     private static final long serialVersionUID = 2L;
-    private ExpressionOperator mCond;
     private LogicalOperator mInput;
+	private LogicalPlan mComparisonPlan;
+	private static Log log = LogFactory.getLog(LOFilter.class);
 
     /**
      * 
@@ -35,29 +38,26 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param cond
      *            the filter condition
      * @param input
      *            the input that needs filtering
      */
-    public LOFilter(LogicalPlan plan, OperatorKey k, int rp,
-            ExpressionOperator cond, LogicalOperator input) {
 
-        super(plan, k, rp);
-        mCond = cond;
+	public LOFilter(LogicalPlan plan, OperatorKey k,
+			LogicalPlan comparisonPlan, LogicalOperator input) {
+		super(plan, k);
+		mComparisonPlan = comparisonPlan;
         mInput = input;
-    }
+	}
 
     public LogicalOperator getInput() {
         return mInput;
     }
 
-    public ExpressionOperator getCondition() {
-        return mCond;
-    }
+	public LogicalPlan getComparisonPlan() {
+		return mComparisonPlan;
+	}
 
     @Override
     public Schema getSchema() throws FrontendException {

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java Thu May  8 14:25:22 2008
@@ -23,6 +23,9 @@
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
 
 public class LOForEach extends LogicalOperator {
 
@@ -34,36 +37,44 @@
      * applied over the input.
      */
 
-    private ArrayList<LogicalOperator> mOperators;
+    private LogicalPlan mForEachPlan;
+    private static Log log = LogFactory.getLog(LOForEach.class);
 
     /**
      * @param plan
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param operators
      *            the list of operators that are applied for each input
      */
-    public LOForEach(LogicalPlan plan, OperatorKey k, int rp,
-            ArrayList<LogicalOperator> operators) {
 
-        super(plan, k, rp);
-        mOperators = operators;
+    public LOForEach(LogicalPlan plan, OperatorKey k,
+            LogicalPlan foreachPlan) {
+
+        super(plan, k);
+        mForEachPlan = foreachPlan;
     }
 
-    public List<LogicalOperator> getOperators() {
-        return mOperators;
+    public LogicalPlan getForEachPlan() {
+        return mForEachPlan;
     }
 
     @Override
     public Schema getSchema() throws FrontendException {
-        if (mSchema == null) {
+        if (!mIsSchemaComputed && (null == mSchema)) {
             // Assuming that the last operator is the GENERATE
             // foreach has to terminate with a GENERATE
-            LogicalOperator last = mOperators.get(mOperators.size() - 1);
+            LogicalOperator last = null;
+			for(LogicalOperator op: mForEachPlan.getLeaves()) {
+				if(op instanceof LOGenerate) {
+					last = op;
+					break;
+				}
+			}
+			if(null == last) throw new FrontendException("Did not find generate in the foreach logicalplan");
+
+            log.debug("Last Operator: " + last.getClass().getName());
             try {
                 mSchema = last.getSchema();
                 mIsSchemaComputed = true;
@@ -78,7 +89,7 @@
 
     @Override
     public String name() {
-        return "Filter " + mKey.scope + "-" + mKey.id;
+        return "ForEach " + mKey.scope + "-" + mKey.id;
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java Thu May  8 14:25:22 2008
@@ -19,12 +19,18 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
 
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.logicalLayer.FrontendException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOGenerate extends LogicalOperator {
     private static final long serialVersionUID = 2L;
@@ -32,7 +38,10 @@
     /**
      * The projection list of this generate.
      */
-    private ArrayList<ExpressionOperator> mProjections;
+    //private ArrayList<ExpressionOperator> mProjections;
+    private ArrayList<LogicalPlan> mGeneratePlans;
+	private ArrayList<Boolean> mFlatten;
+    private static Log log = LogFactory.getLog(LOGenerate.class);
 
     /**
      * 
@@ -40,22 +49,45 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param projections
      *            the projection list of the generate
      */
-    public LOGenerate(LogicalPlan plan, OperatorKey key, int rp,
-            ArrayList<ExpressionOperator> projections) {
-        super(plan, key, rp);
-        mProjections = projections;
+
+    public LOGenerate(LogicalPlan plan, OperatorKey key,
+            ArrayList<LogicalPlan> generatePlans, ArrayList<Boolean> flatten) {
+        super(plan, key);
+        mGeneratePlans = generatePlans;
+        mFlatten = flatten;
+    }
+
+    /**
+     * 
+     * @param plan
+     *            Logical plan this operator is a part of.
+     * @param k
+     *            Operator key to assign to this node.
+     * @param projection
+     *            the projection of the generate
+     */
+
+    public LOGenerate(LogicalPlan plan, OperatorKey key,
+            LogicalPlan generatePlan, boolean flatten) {
+        super(plan, key);
+        mGeneratePlans = new ArrayList<LogicalPlan>();
+        mGeneratePlans.add(generatePlan);
+        mFlatten = new ArrayList<Boolean>();
+		mFlatten.add(flatten);
     }
 
-    public List<ExpressionOperator> getProjections() {
-        return mProjections;
+
+    public List<LogicalPlan> getGeneratePlans() {
+        return mGeneratePlans;
     }
 
+	public List<Boolean> getFlatten() {
+		return mFlatten;
+	}
+
     @Override
     public String name() {
         return "Generate " + mKey.scope + "-" + mKey.id;
@@ -73,26 +105,95 @@
 
     @Override
     public Schema getSchema() throws FrontendException {
-        if (mSchema == null) {
+        log.debug("Entering getSchema");
+        if (!mIsSchemaComputed && (null == mSchema)) {
             List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(
-                    mProjections.size());
-            for (ExpressionOperator op : mProjections) {
-                String opAlias = op.getAlias();
-                if (op.getType() == DataType.TUPLE) {
-                    try {
-                        fss.add(new Schema.FieldSchema(opAlias, op.getSchema()));
-                    } catch (FrontendException ioe) {
-                        mSchema = null;
-                        mIsSchemaComputed = false;
-                        throw ioe;
+                    mGeneratePlans.size());
+
+            for (LogicalPlan plan : mGeneratePlans) {
+				log.debug("Number of roots in " + plan + " = " + plan.getRoots().size());
+				for(int i = 0; i < plan.getRoots().size(); ++i) {
+					log.debug("Root" + i + "= " + plan.getRoots().get(i));
+				}
+				LogicalOperator op = plan.getRoots().get(0);
+                log.debug("op: " + op.getClass().getName() + " " + op);
+			}
+			log.debug("Printed the roots of the generate plans");
+
+			Map<Schema.FieldSchema, String> flattenAlias = new HashMap<Schema.FieldSchema, String>();
+
+            for (int planCtr = 0; planCtr < mGeneratePlans.size(); ++planCtr) {
+				LogicalPlan plan = mGeneratePlans.get(planCtr);
+				LogicalOperator op = plan.getRoots().get(0);
+                log.debug("op: " + op.getClass().getName() + " " + op);
+                Set<String> aliases;
+                Iterator<String> iter = null;
+                String opSchemaAlias = null;
+                Schema s = null;
+
+                try {
+                    s = op.getSchema();
+                    if (null != s) {
+                        log.debug("Printing aliases in LOGenerate");
+                        s.printAliases();
+                        aliases = op.getSchema().getAliases();
+                        iter = aliases.iterator();
+                    }
+                } catch (FrontendException ioe) {
+                    mSchema = null;
+                    mIsSchemaComputed = false;
+                    throw ioe;
+                }
+
+                // ASSUMPTION
+                // Here I am assuming that the LOProject does not have multiple
+                // columns in the project, i.e., generate A.($1,$2,$3) as
+                // (name,age,gpa)
+
+                if ((null != iter) && (iter.hasNext())) {
+                    opSchemaAlias = iter.next();
+                    log.debug("iter.next: " + opSchemaAlias);
+                }
+                
+
+                log.debug("Type: " + DataType.findTypeName(op.getType())
+                        + " Alias: " + opSchemaAlias);
+
+                if (mFlatten.get(planCtr)) {
+                    log.debug("Flatten");
+                    for(Schema.FieldSchema fs: s.getFields()) {
+						Schema internalSchema = fs.schema;
+						Schema.FieldSchema newFs;
+						log.debug("Flatten level 1 internalSchema: " + internalSchema);
+						if(null != internalSchema) {
+							for(Schema.FieldSchema internalfs: internalSchema.getFields()) {
+                        		newFs = new Schema.FieldSchema(internalfs.alias, internalfs.schema);
+								fss.add(newFs);
+								log.debug("Flatten alias: " + opSchemaAlias+"::"+internalfs.alias);
+								flattenAlias.put(newFs, opSchemaAlias+"::"+internalfs.alias);
+							}
+						}
+						else {
+                        	newFs= new Schema.FieldSchema(fs.alias, fs.schema);
+                        	fss.add(newFs);
+							log.debug("Flatten alias: " + opSchemaAlias+"::"+fs.alias);
+							flattenAlias.put(newFs, opSchemaAlias+"::"+fs.alias);
+						}
                     }
                 } else {
-                    fss.add(new Schema.FieldSchema(opAlias, op.getType()));
+                    fss.add(new Schema.FieldSchema(opSchemaAlias, s));
                 }
             }
             mSchema = new Schema(fss);
+			for(Schema.FieldSchema fs: mSchema.getFields()) {
+				String alias = flattenAlias.get(fs);
+				if(null != alias) {
+					mSchema.addAlias(alias, fs);
+				}
+			}
             mIsSchemaComputed = true;
         }
+        log.debug("Exiting getSchema");
         return mSchema;
     }
 

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOGreaterThan extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOGreaterThan.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOGreaterThan(LogicalPlan plan, OperatorKey k, int rp,
+    public LOGreaterThan(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOGreaterThanEqual extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOGreaterThanEqual.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOGreaterThanEqual(LogicalPlan plan, OperatorKey k, int rp,
+    public LOGreaterThanEqual(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOLesserThan extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOLesserThan.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOLesserThan(LogicalPlan plan, OperatorKey k, int rp,
+    public LOLesserThan(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOLesserThanEqual extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOLesserThanEqual.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOLesserThanEqual(LogicalPlan plan, OperatorKey k, int rp,
+    public LOLesserThanEqual(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java Thu May  8 14:25:22 2008
@@ -24,11 +24,14 @@
 import java.util.Map;
 import java.net.URL;
 
-import org.apache.pig.LoadFunc; // import org.apache.pig.impl.PigContext;
+import org.apache.pig.LoadFunc; 
+import org.apache.pig.impl.PigContext;
 import org.apache.pig.impl.io.FileSpec;
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOLoad extends LogicalOperator {
     private static final long serialVersionUID = 2L;
@@ -36,36 +39,34 @@
     private FileSpec mInputFileSpec;
     private LoadFunc mLoadFunc;
     private URL mSchemaFile;
+	private static Log log = LogFactory.getLog(LOLoad.class);
 
     /**
      * @param plan
      *            LogicalPlan this operator is a part of.
      * @param key
      *            OperatorKey for this operator
-     * @param rp
-     *            Requested level of parallelism to be used in the sort.
      * @param inputFileSpec
-     *            the file to be loaded
+     *            the file to be loaded *
+     * @param schemaFile
+     *            the file with the schema for the data to be loaded
+     * 
      */
-    public LOLoad(LogicalPlan plan, OperatorKey key, int rp,
-            FileSpec inputFileSpec, URL schemaFile) throws IOException {
-        super(plan, key, rp);
+    public LOLoad(LogicalPlan plan, OperatorKey key, FileSpec inputFileSpec,
+            URL schemaFile) throws IOException {
+        super(plan, key);
 
         mInputFileSpec = inputFileSpec;
         mSchemaFile = schemaFile;
 
-        // TODO FIX
-        // The code below is commented out as PigContext pulls in
-        // HExecutionEngine which in turn is completely commented out
-        // Also remove the commented out import org.apache.pig.impl.PigContext
-
-        /*
-         * try { mLoadFunc = (LoadFunc)
-         * PigContext.instantiateFuncFromSpec(inputFileSpec.getFuncSpec()); }
-         * catch (Exception e){ IOException ioe = new
-         * IOException(e.getMessage()); ioe.setStackTrace(e.getStackTrace());
-         * throw ioe; }
-         */
+         try { 
+		 	mLoadFunc = (LoadFunc)
+          	    PigContext.instantiateFuncFromSpec(inputFileSpec.getFuncSpec()); 
+		} catch (Exception e){ 
+			IOException ioe = new IOException(e.getMessage()); 
+			ioe.setStackTrace(e.getStackTrace());
+        	throw ioe; 
+		}
     }
 
     public FileSpec getInputFile() {
@@ -90,11 +91,17 @@
         if (!mIsSchemaComputed && (null == mSchema)) {
             // get the schema of the load function
             try {
-                mSchema = mLoadFunc.determineSchema(mSchemaFile);
+                //DEBUG
+                //System.out.println("Schema file: " + mSchema);
+                if(null != mSchemaFile) {
+                    mSchema = mLoadFunc.determineSchema(mSchemaFile);
+                }
                 mIsSchemaComputed = true;
             } catch (IOException ioe) {
                 FrontendException fee = new FrontendException(ioe.getMessage());
                 fee.initCause(ioe);
+                mIsSchemaComputed = false;
+                mSchema = null;
                 throw fee;
             }
         }

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java Thu May  8 14:25:22 2008
@@ -25,6 +25,8 @@
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOMapLookup extends ExpressionOperator {
     private static final long serialVersionUID = 2L;
@@ -35,8 +37,9 @@
      */
     private ExpressionOperator mMap;
     private Object mMapKey;
-    private DataType mValueType;
+    private byte mValueType;
     private Schema mValueSchema;
+    private static Log log = LogFactory.getLog(LOMapLookup.class);
 
     /**
      * 
@@ -44,9 +47,6 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param map
      *            the map expression
      * @param mapKey
@@ -56,10 +56,10 @@
      * @param valueSchema
      *            schema of the value if the type is tuple
      */
-    public LOMapLookup(LogicalPlan plan, OperatorKey key, int rp, ExpressionOperator map,
-            Object mapKey, DataType valueType, Schema valueSchema)
+    public LOMapLookup(LogicalPlan plan, OperatorKey key, ExpressionOperator map,
+            Object mapKey, byte valueType, Schema valueSchema)
             throws ParseException {
-        super(plan, key, rp);
+        super(plan, key);
 
         if (!DataType.isAtomic(mapKey)) {
             throw new ParseException("Map key" + mapKey.toString()
@@ -79,7 +79,7 @@
         return mMapKey;
     }
 
-    public DataType getValueType() {
+    public byte getValueType() {
         return mValueType;
     }
 

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOMod extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOMod.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOMod(LogicalPlan plan, OperatorKey k, int rp,
+    public LOMod(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOMultiply extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOMultiply.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOMultiply(LogicalPlan plan, OperatorKey k, int rp,
+    public LOMultiply(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java Thu May  8 14:25:22 2008
@@ -32,15 +32,12 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param operand
      *            the only operand for a unary operator
      */
-    public LONegative(LogicalPlan plan, OperatorKey k, int rp,
+    public LONegative(LogicalPlan plan, OperatorKey k,
             ExpressionOperator operand) {
-        super(plan, k, rp, operand);
+        super(plan, k, operand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LONot extends UnaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LONot.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param operand
      *            the only operand for a unary operator
      */
-    public LONot(LogicalPlan plan, OperatorKey k, int rp,
+    public LONot(LogicalPlan plan, OperatorKey k,
             ExpressionOperator operand) {
-        super(plan, k, rp, operand);
+        super(plan, k, operand);
     }
-
+    
     @Override
     public Schema getSchema() {
         return mSchema;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LONotEqual extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LONotEqual.class);
 
     /**
      * 
@@ -32,17 +35,14 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LONotEqual(LogicalPlan plan, OperatorKey k, int rp,
+    public LONotEqual(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
 
     @Override

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java Thu May  8 14:25:22 2008
@@ -21,10 +21,13 @@
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.plan.PlanVisitor;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
 public class LOOr extends BinaryExpressionOperator {
 
     private static final long serialVersionUID = 2L;
+    private static Log log = LogFactory.getLog(LOOr.class);
 
     /**
      * 
@@ -32,19 +35,16 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param lhsOperand
      *            the left hand side operand
      * @param rhsOperand
      *            the right hand side operand
      */
-    public LOOr(LogicalPlan plan, OperatorKey k, int rp,
+    public LOOr(LogicalPlan plan, OperatorKey k,
             ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
-        super(plan, k, rp, lhsOperand, rhsOperand);
+        super(plan, k, lhsOperand, rhsOperand);
     }
-
+    
     @Override
     public Schema getSchema() {
         return mSchema;

Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java Thu May  8 14:25:22 2008
@@ -21,19 +21,47 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 import org.apache.pig.data.DataType;
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.impl.logicalLayer.schema.Schema;
 import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
 
+/**
+ * LOProject is designed like a singly linked list; A few examples will
+ * illustrate the point about the linked list nature of the design;
+ * a = load 'input1' as (name, age);
+ * b = group a by name;
+ * foreach b generate a, a.name;
+ * The project operator occurs in two places in the above script:
+ * generate a(here) and a.name(here)
+ * In the first occurrence, we are trying to project the elements of
+ * the bag a; In order to retrieve the bag, we need to project the
+ * the second column ($1) or column number 1 (using the zero based index)
+ * from the input (the relation or bag b)
+ * In the second occurence, we are trying to project the first column
+ * ($0) or column number 0 from the bag a which in turn is the column
+ * number 1 in the relation b; As you can see, the nested structure or
+ * the singly linked list nature is clearly visible;
+ * Given that it's a singly linked list, the null pointer or the sentinel
+ * is marked explictly using the boolean variable mSentinel; The sentinel
+ * is marked true only when the input is a relational operator; This occurs
+ * when we create the innermost operator
+ */
 public class LOProject extends ExpressionOperator {
     private static final long serialVersionUID = 2L;
 
     /**
      * The expression and the column to be projected.
      */
-    private ExpressionOperator mExp;
-    private List<String> mProjection;
+    private LogicalOperator mExp;
+    private List<Integer> mProjection;
+    private boolean mIsStar = false;
+    private static Log log = LogFactory.getLog(LOProject.class);
+	private boolean mSentinel;
 
     /**
      * 
@@ -41,19 +69,21 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param exp
      *            the expression which might contain the column to project
      * @param projection
      *            the list of columns to project
      */
-    public LOProject(LogicalPlan plan, OperatorKey key, int rp,
-            ExpressionOperator exp, List<String> projection) {
-        super(plan, key, rp);
+    public LOProject(LogicalPlan plan, OperatorKey key, LogicalOperator exp,
+            List<Integer> projection) {
+        super(plan, key);
         mExp = exp;
         mProjection = projection;
+		if(mExp instanceof ExpressionOperator) {
+			mSentinel = false;
+		} else {
+			mSentinel = true;
+		}
     }
 
     /**
@@ -62,30 +92,53 @@
      *            Logical plan this operator is a part of.
      * @param k
      *            Operator key to assign to this node.
-     * @param rp
-     *            degree of requested parallelism with which to execute this
-     *            node.
      * @param exp
      *            the expression which might contain the column to project
      * @param projection
      *            the column to project
      */
-    public LOProject(LogicalPlan plan, OperatorKey key, int rp,
-            ExpressionOperator exp, String projection) {
-        super(plan, key, rp);
+    public LOProject(LogicalPlan plan, OperatorKey key, LogicalOperator exp,
+            Integer projection) {
+        super(plan, key);
         mExp = exp;
-        mProjection = new ArrayList<String>(1);
+        mProjection = new ArrayList<Integer>(1);
         mProjection.add(projection);
+		if(mExp instanceof ExpressionOperator) {
+			mSentinel = false;
+		} else {
+			mSentinel = true;
+		}
     }
 
-    public ExpressionOperator getExpression() {
+    public LogicalOperator getExpression() {
         return mExp;
     }
 
-    public List<String> getProjection() {
+    public List<Integer> getProjection() {
         return mProjection;
     }
 
+    public int getCol() {
+        if (mProjection.size() != 1)
+            throw new RuntimeException(
+                    "Internal error: improper use of getCol in "
+                            + LOProject.class.getName());
+        return mProjection.get(0);
+
+    }
+
+    public void setStar(boolean b) {
+        mIsStar = b;
+    }
+
+    public boolean getSentinel() {
+        return mSentinel;
+    }
+
+    public void setSentinel(boolean b) {
+        mSentinel = b;
+    }
+
     @Override
     public String name() {
         return "Project " + mKey.scope + "-" + mKey.id;
@@ -97,7 +150,159 @@
     }
 
     @Override
-    public Schema getSchema() {
+    public Schema getSchema() throws FrontendException {
+        log.debug("Inside getSchema");
+		log.debug("Number of columns: " + mProjection.size());
+        for (int i : mProjection) {
+            log.debug("Column: " + i);
+        }
+		LogicalOperator expressionOperator = mExp;
+		log.debug("expressionOperator = " + expressionOperator);
+		log.debug("mIsStar: " + mIsStar);
+
+        if (!mIsSchemaComputed && (null == mSchema)) {
+
+            if (mIsStar) {
+                log.debug("mIsStar is true");
+                try {
+                    if (null != expressionOperator) {
+                        log.debug("expressionOperator is not null "
+                                + expressionOperator.getClass().getName() + " " + expressionOperator);
+                        mSchema = expressionOperator.getSchema();
+                    }
+                    mIsSchemaComputed = true;
+                } catch (FrontendException ioe) {
+                    mSchema = null;
+                    mIsSchemaComputed = false;
+                    throw ioe;
+                }
+                log.debug("mIsStar is true, returning schema of expressionOperator");
+                log.debug("Exiting getSchema()");
+                return mSchema;
+            } else {
+                List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(
+                        mProjection.size());
+
+                if (mProjection.size() == 1) {
+                    log.debug("Only one element");
+                    try {
+                        Schema s = expressionOperator.getSchema();
+                        if (null != s) {
+							log.debug("Getting the fieldschema for column: " + mProjection.get(0));
+                            Schema.FieldSchema fs = null; 
+							if(!mSentinel) {
+								log.debug("We are in the outer part of the projection");
+								fs = s.getField(0);
+								log.debug("fs in outer part: " + fs);
+								s = fs.schema;
+								log.debug("s: " + s);
+								if(null != s) {
+									fs = s.getField(mProjection.get(0));
+									log.debug("fs in outer part after unwrapping: " + fs);
+								} else {
+									fs = null;
+									log.debug("fs in outer part after unwrapping: " + fs);
+								}
+							} else {
+								log.debug("We are in the inner most part of the projection");
+								fs = s.getField(mProjection.get(0));
+							}
+                            if (null != fs) {
+                                log.debug("fs.type: "
+                                        + DataType.findTypeName(fs.type)
+                                        + " fs.schema: " + fs.schema);
+                                if (fs.type == DataType.BAG
+                                        || fs.type == DataType.TUPLE) {
+                                    if (null != fs.schema) {
+                                        log.debug("fs.schema aliases");
+                                        fs.schema.printAliases();
+                                    }
+                                    
+                                    //mSchema = fs.schema;
+                                    mSchema = new Schema(new Schema.FieldSchema(fs.alias, fs.schema));
+                                    //mSchema = new Schema(fs);
+                                } else {
+                                    mSchema = new Schema(fs);
+                                }
+                            } else {
+                            	log.debug("expressionOperator.schema is null");
+                            	mSchema = new Schema(new Schema.FieldSchema(null,
+                                    DataType.BYTEARRAY));
+                            }
+                        } else {
+                            log.debug("expressionOperator.schema is null");
+                            mSchema = new Schema(new Schema.FieldSchema(null,
+                                    DataType.BYTEARRAY));
+                        }
+                    } catch (Exception e) {
+                        mSchema = null;
+                        mIsSchemaComputed = false;
+                        throw new FrontendException(e.getMessage());
+                    }
+                    mIsSchemaComputed = true;
+                    log.debug("Exiting getSchema");
+                    return mSchema;
+                }
+
+                if (null != expressionOperator) {
+                    log.debug("expressionOperator is not null");
+
+                    Schema s = expressionOperator.getSchema();
+                    log.debug("s: " + s);
+                    for (int colNum : mProjection) {
+                        log.debug("Col: " + colNum);
+                        if (null != s) {
+                            try {
+                                Schema.FieldSchema tempFs = s.getField(0);
+                                if (null != tempFs) {
+                                    if(null != tempFs.schema) {
+                                        Schema.FieldSchema fs = tempFs.schema.getField(colNum);
+                                        log.debug("fs.type: "
+                                                + DataType.findTypeName(fs.type)
+                                                + " fs.schema: " + fs.schema);
+                                        if(fs.type == DataType.BAG || fs.type == DataType.TUPLE) {
+                                            if (null != fs.schema) {
+                                                log.debug("fs.schema aliases");
+                                                fs.schema.printAliases();
+                                            }
+                                                fss.add(new Schema.FieldSchema(fs.alias, fs.schema));
+                                        } else {
+                                            fss.add(fs);
+                                        }
+                                    } else {
+                                        log.debug("expressionOperator.fs.schema is null");
+                                        fss.add(new Schema.FieldSchema(null,
+                                                DataType.BYTEARRAY));
+                                    }
+                                } else {
+									log.debug("Could not find column " + colNum+ " in schema");
+                                    throw new FrontendException(
+                                            "Could not find column " + colNum
+                                                    + " in schema");
+                                }
+                            } catch (Exception e) {
+								log.debug("Caught exception: " + e.getMessage());
+                                mSchema = null;
+                                mIsSchemaComputed = false;
+                                throw new FrontendException(e.getMessage());
+                            }
+                        } else {
+                            log.debug("expressionOperator.schema is null");
+                            fss.add(new Schema.FieldSchema(null,
+                                    DataType.BYTEARRAY));
+                        }
+                    }
+                } else {
+                    log.debug("expressionOperator is null");
+                    fss.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
+                }
+                mSchema = new Schema(fss);
+                mIsSchemaComputed = true;
+                log.debug("mIsStar is false, returning computed schema of expressionOperator");
+            }
+        }
+
+        log.debug("Exiting getSchema");
         return mSchema;
     }