You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ga...@apache.org on 2008/05/08 23:25:54 UTC
svn commit: r654629 [1/4] - in /incubator/pig/branches/types: ./
src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/logicalLayer/
src/org/apache/pig/impl/logicalLayer/parser/
src/org/apache/pig/impl/logicalLayer/schema/ src/org/apache/pig/impl/logic...
Author: gates
Date: Thu May 8 14:25:22 2008
New Revision: 654629
URL: http://svn.apache.org/viewvc?rev=654629&view=rev
Log:
Santhosh's massive parser rewrite.
Added:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplitOutput.java
Removed:
incubator/pig/branches/types/src/org/apache/pig/impl/eval/BinCondSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/CompositeEvalSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/ConstSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/EvalSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/EvalSpecPrinter.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/EvalSpecVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/FilterSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/FuncEvalSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/GenerateSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/MapLookupSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/ProjectSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/SimpleEvalSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/SortDistinctSpec.java
incubator/pig/branches/types/src/org/apache/pig/impl/eval/StarSpec.java
Modified:
incubator/pig/branches/types/build.xml
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSplit.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOStore.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSubtract.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUnion.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOUserFunc.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlan.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LogicalPlanBuilder.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/UnaryExpressionOperator.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/schema/Schema.java
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
Modified: incubator/pig/branches/types/build.xml
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Thu May 8 14:25:22 2008
@@ -144,7 +144,7 @@
**/test/FakeFSOutputStream.java, **/test/TestPackage.java, **/test/TestForEach.java,
**/test/TestLocalRearrange.java, **/test/TestPOUserFunc.java,
**/test/TestPODistinct.java, **/test/TestPOSort.java,
- **/test/TestSchema.java,
+ **/test/TestSchema.java, **/test/TestLogicalPlanBuilder.java,
**/test/FakeFSInputStream.java, **/test/Util.java,
**/logicalLayer/*.java, **/logicalLayer/parser/NodeIdGenerator.java,
**/logicalLayer/schema/*.java, **/physicalLayer/topLevelOperators/*.java,
@@ -265,6 +265,7 @@
<include name="**/TestInputOutputFileValidator.java" />
<include name="**/TestTypeCheckingValidator.java" />
<include name="**/TestSchema.java" />
+ <include name="**/TestLogicalPlanBuilder.java" />
<!--
<include name="**/*Test*.java" />
<exclude name="**/TestLargeFile.java" />
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/BinaryExpressionOperator.java Thu May 8 14:25:22 2008
@@ -20,6 +20,8 @@
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* This abstract class represents the logical Binary Expression Operator
@@ -32,6 +34,7 @@
private static final long serialVersionUID = 2L;
private ExpressionOperator mLhsOperand; //left hand side operand
private ExpressionOperator mRhsOperand; //right hand side operand
+ private static Log log = LogFactory.getLog(BinaryExpressionOperator.class);
/**
* @param plan
@@ -53,6 +56,23 @@
mRhsOperand = rhsOperand;
}
+ /**
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
+ * @param lhsOperand
+ * ExpressionOperator the left hand side operand
+ * @param rhsOperand
+ * ExpressionOperator the right hand side operand
+ */
+ public BinaryExpressionOperator(LogicalPlan plan, OperatorKey k,
+ ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
+ super(plan, k);
+ mLhsOperand = lhsOperand;
+ mRhsOperand = rhsOperand;
+ }
+
public ExpressionOperator getLhsOperand() {
return mLhsOperand;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/ExpressionOperator.java Thu May 8 14:25:22 2008
@@ -19,10 +19,13 @@
import java.util.List;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public abstract class ExpressionOperator extends LogicalOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(ExpressionOperator.class);
/**
* @param plan
@@ -37,6 +40,17 @@
super(plan, k, rp);
}
+ /**
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
+ */
+ public ExpressionOperator(LogicalPlan plan, OperatorKey k) {
+ super(plan, k);
+ }
+
+
@Override
public boolean supportsMultipleOutputs() {
return false;
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAdd.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOAdd extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOAdd.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOAdd(LogicalPlan plan, OperatorKey k, int rp,
+ public LOAdd(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOAnd.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOAnd extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOAnd.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOAnd(LogicalPlan plan, OperatorKey k, int rp,
+ public LOAnd(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOBinCond.java Thu May 8 14:25:22 2008
@@ -41,9 +41,6 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param cond
* ExpressionOperator the expression specifying condition
* @param lhsOp
@@ -52,10 +49,10 @@
* ExpressionOperator query to be executed when condition is
* false
*/
- public LOBinCond(LogicalPlan plan, OperatorKey k, int rp,
+ public LOBinCond(LogicalPlan plan, OperatorKey k,
ExpressionOperator cond, ExpressionOperator lhsOp,
ExpressionOperator rhsOp) {
- super(plan, k, rp);
+ super(plan, k);
mCond = cond;
mLhsOp = lhsOp;
mRhsOp = rhsOp;
@@ -89,9 +86,12 @@
@Override
public Schema getSchema() throws FrontendException {
+ //TODO
+ //We need a check of LHS and RHS schemas
if (!mIsSchemaComputed && (null == mSchema)) {
try {
mSchema = mLhsOp.getSchema();
+ //mSchema = mLhsPlan.getRoots().get(0).getSchema();
mIsSchemaComputed = true;
} catch (FrontendException ioe) {
mSchema = null;
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCast.java Thu May 8 14:25:22 2008
@@ -35,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param expr
* the expression whose type has to be cast
* @param type
* the type to which the expression is cast
*/
- public LOCast(LogicalPlan plan, OperatorKey k, int rp,
+ public LOCast(LogicalPlan plan, OperatorKey k,
ExpressionOperator expr, byte type) {
- super(plan, k, rp);
+ super(plan, k);
mExpr = expr;
mType = type;
}// End Constructor LOCast
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCogroup.java Thu May 8 14:25:22 2008
@@ -18,20 +18,35 @@
package org.apache.pig.impl.logicalLayer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.HashMap;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.impl.plan.MultiMap;
public class LOCogroup extends LogicalOperator {
private static final long serialVersionUID = 2L;
- private ArrayList<String> mInputs;
- private ArrayList<ExpressionOperator> mGroupByCols;
+ /**
+ * Cogroup contains a list of logical operators corresponding to the
+ * relational operators and a list of generates for each relational
+ * operator. Each generate operator in turn contains a list of expressions
+ * for the columns that are projected
+ */
+ private ArrayList<LogicalOperator> mInputs;
+ private boolean[] mIsInner;
+ private static Log log = LogFactory.getLog(LOCogroup.class);
+ private MultiMap<LogicalOperator, LogicalPlan> mGroupByPlans;
/**
*
@@ -39,24 +54,28 @@
* LogicalPlan this operator is a part of.
* @param key
* OperatorKey for this operator
- * @param rp
- * Requested level of parallelism to be used
* @param groupByCols
* the group by columns
*/
- public LOCogroup(LogicalPlan plan, OperatorKey k, int rp,
- ArrayList<String> inputs, ArrayList<ExpressionOperator> groupByCols) {
- super(plan, k, rp);
+ public LOCogroup(LogicalPlan plan, OperatorKey k,
+ ArrayList<LogicalOperator> inputs,
+ MultiMap<LogicalOperator, LogicalPlan> groupByPlans, boolean[] isInner) {
+ super(plan, k);
mInputs = inputs;
- mGroupByCols = groupByCols;
+ mGroupByPlans = groupByPlans;
+ mIsInner = isInner;
}
- public List<String> getInputs() {
+ public List<LogicalOperator> getInputs() {
return mInputs;
}
- public List<ExpressionOperator> getGroupByCols() {
- return mGroupByCols;
+ public MultiMap<LogicalOperator, LogicalPlan> getGroupByPlans() {
+ return mGroupByPlans;
+ }
+
+ public boolean[] getInner() {
+ return mIsInner;
}
@Override
@@ -77,7 +96,7 @@
@Override
public Schema getSchema() throws FrontendException {
// TODO create schema
- /**
+ /*
* Dumping my understanding of how the schema of a Group/CoGroup will
* look. The first field of the resulting tuple will have the alias
* 'group'. The schema for this field is a union of the group by columns
@@ -85,19 +104,150 @@
* the alias of the input as the alias for a bag that contains the
* tuples from the input that match the grouping criterion
*/
- if (null == mSchema) {
+ if (!mIsSchemaComputed && (null == mSchema)) {
List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(
- mGroupByCols.size() + 1);
+ mInputs.size() + 1);
// one more to account for the "group"
// the alias of the first field is group and hence the
// string "group"
// TODO The type of the field named "group" requires
// type promotion and the like
- fss.add(new Schema.FieldSchema("group", null));
- for (String input : mInputs) {
- fss.add(new Schema.FieldSchema(input, DataType.BAG));
+
+ /*
+ * Here goes an attempt to describe how the schema for the first
+ * column - 'group' should look like If the number of group by
+ * columns = 1 then the schema for 'group' is the
+ * schema(fieldschema(col)) If the number of group by columns > 1
+ * then find the set union of the group by columns and form the
+ * schema as schema(list<fieldschema of the cols>)
+ * The parser will ensure that the number of group by columns are
+ * the same across all inputs. The computation of the schema for group
+ * is
+ */
+
+ Schema groupBySchema = null;
+ List<Schema.FieldSchema> groupByFss = new ArrayList<Schema.FieldSchema>();
+ Set<String> groupByAliases = new HashSet<String>();
+ Map<String, Boolean> lookup = new HashMap<String, Boolean>();
+ MultiMap<String, ExpressionOperator> aliasExop = new MultiMap<String, ExpressionOperator>();
+ MultiMap<Integer, String> positionAlias= new MultiMap<Integer, String>();
+
+ for (LogicalOperator op : mInputs) {
+ log.debug("GBY Input: " + op.getClass().getName());
+ for(LogicalPlan plan: mGroupByPlans.get(op)) {
+ int position = 0;
+ for(LogicalOperator eOp: plan.getRoots()) {
+ Schema eOpSchema = eOp.getSchema();
+ log.debug("Computing the lookup tables");
+ if (null != eOpSchema) {
+ for (String alias : eOpSchema.getAliases()) {
+ log.debug("Adding alias to GBY: " + alias);
+ groupByAliases.add(alias);
+ lookup.put(alias, false);
+ aliasExop.put(alias, (ExpressionOperator)eOp);
+ positionAlias.put(position, alias);
+ }
+ }
+ }
+ ++position;
+ }
+ }
+
+ log.debug("Computed the lookup table");
+
+ int arity = mGroupByPlans.get(mInputs.get(0)).size();
+ log.debug("Arity: " + arity);
+ for (int i = 0; i < arity; ++i) {
+ Collection<String> cAliases;
+ cAliases = positionAlias.get(i);
+ if(null != cAliases) {
+ Object[] aliases = cAliases.toArray();
+ for(int j = 0; j < aliases.length; ++j) {
+ String alias = (String) aliases[j];
+ if(null != alias) {
+ Collection<ExpressionOperator> cEops = aliasExop.get(alias);
+ if(null != cEops) {
+ ExpressionOperator eOp = (ExpressionOperator) (cEops.toArray())[0];
+ if(null != eOp) {
+ if(!lookup.get(alias)) {
+ groupByFss.add(new Schema.FieldSchema(alias, eOp.getSchema()));
+ lookup.put(alias, true);
+ } else {
+ if(j < aliases.length) {
+ continue;
+ } else {
+ //we have seen this alias before
+ //just add the schema of the expression operator with the null alias
+ groupByFss.add(new Schema.FieldSchema(null, eOp.getSchema()));
+ break;
+ }
+ }
+ } else {
+ //should not be here
+ log.debug("Cannot be here: we cannot have a collection of null expression operators");
+ }
+ } else {
+ //should not be here
+ log.debug("Cannot be here: we cannot have an alias without an expression operator");
+ }
+ } else {
+ //should not be here
+ log.debug("Cannot be here: we cannot have a collection of null aliases ");
+ }
+ }
+ } else {
+ //We do not have any alias for this position in the group by columns
+ //We have positions $1, $2, etc.
+ //The schema for these columns is the schema of the expression operatore
+ //and so the alias is null
+ groupByFss.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
+ }
+ }
+
+ groupBySchema = new Schema(groupByFss);
+ log.debug("Printing group by schema aliases");
+ groupBySchema.printAliases();
+
+ if(1 == arity) {
+ log.debug("Arity == 1");
+ byte groupByType = groupByFss.get(0).type;
+ Schema groupSchema = groupByFss.get(0).schema;
+ log.debug("Type == " + DataType.findTypeName(groupByType));
+ if(DataType.TUPLE == groupByType || DataType.BAG == groupByType) {
+ if(null != groupSchema) {
+ Schema innerSchema = groupSchema.getFields().get(0).schema;
+ fss.add(new Schema.FieldSchema("group", innerSchema, groupByType));
+ } else {
+ fss.add(new Schema.FieldSchema("group", groupSchema, groupByType));
+ }
+ log.debug("Printing the aliases of the single group by column");
+ groupSchema.printAliases();
+ } else {
+ fss.add(new Schema.FieldSchema("group", groupByType));
+ }
+ } else {
+ fss.add(new Schema.FieldSchema("group", groupBySchema));
+ }
+ for (LogicalOperator op : mInputs) {
+ log.debug("Op: " + op.getClass().getName());
+ log.debug("Op Alias: " + op.getAlias());
+ try {
+ Schema cSchema = op.getSchema();
+ if (null != cSchema) {
+ log.debug("Printing constituent schema aliases");
+ cSchema.printAliases();
+ }
+ fss.add(new Schema.FieldSchema(op.getAlias(), op
+ .getSchema(), DataType.BAG));
+ } catch (FrontendException ioe) {
+ log.debug("Caught an exception: " + ioe.getMessage());
+ mIsSchemaComputed = false;
+ mSchema = null;
+ throw ioe;
+ }
}
mIsSchemaComputed = true;
+ mSchema = new Schema(fss);
}
return mSchema;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOConst.java Thu May 8 14:25:22 2008
@@ -35,14 +35,11 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param value
* the value of the constant
*/
- public LOConst(LogicalPlan plan, OperatorKey k, int rp, Object value) {
- super(plan, k, rp);
+ public LOConst(LogicalPlan plan, OperatorKey k, Object value) {
+ super(plan, k);
mValue = value;
}// End Constructor LOConst
Added: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java?rev=654629&view=auto
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java (added)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOCross.java Thu May 8 14:25:22 2008
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.io.IOException;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class LOCross extends LogicalOperator {
+
+ private static final long serialVersionUID = 2L;
+ private ArrayList<LogicalOperator> mInputs;
+ private static Log log = LogFactory.getLog(LOCross.class);
+
+ /**
+ *
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
+ */
+ public LOCross(LogicalPlan plan, OperatorKey k, ArrayList<LogicalOperator> inputs) {
+
+ super(plan, k);
+ mInputs = inputs;
+ }
+
+ public List<LogicalOperator> getInputs() {
+ return mInputs;
+ }
+
+ public void addInput(LogicalOperator input) {
+ mInputs.add(input);
+ }
+
+ @Override
+ public Schema getSchema() throws FrontendException {
+ if (!mIsSchemaComputed && (null == mSchema)) {
+ // Get the schema of the parents
+ Collection<LogicalOperator> s = mPlan.getPredecessors(this);
+ List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(s
+ .size());
+ for (LogicalOperator op : s) {
+ String opAlias = op.getAlias();
+ if (op.getType() == DataType.TUPLE) {
+ try {
+ fss.add(new Schema.FieldSchema(opAlias, op
+ .getSchema()));
+ } catch (FrontendException ioe) {
+ mSchema = null;
+ mIsSchemaComputed = false;
+ throw ioe;
+ }
+ } else {
+ fss.add(new Schema.FieldSchema(opAlias, op.getType()));
+ }
+ }
+ mSchema = new Schema(fss);
+ mIsSchemaComputed = true;
+ }
+ return mSchema;
+ }
+
+ @Override
+ public String name() {
+ return "Cross " + mKey.scope + "-" + mKey.id;
+ }
+
+ @Override
+ public boolean supportsMultipleInputs() {
+ return true;
+ }
+
+ @Override
+ public boolean supportsMultipleOutputs() {
+ return false;
+ }
+
+ @Override
+ public void visit(LOVisitor v) throws VisitorException {
+ v.visit(this);
+ }
+
+}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODistinct.java Thu May 8 14:25:22 2008
@@ -25,10 +25,14 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LODistinct extends LogicalOperator {
private static final long serialVersionUID = 2L;
+ private LogicalOperator mInput;
+ private static Log log = LogFactory.getLog(LODistinct.class);
/**
*
@@ -36,23 +40,28 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
+ * @param input
+ * input over which distinct will be applied
*/
- public LODistinct(LogicalPlan plan, OperatorKey k, int rp) {
+ public LODistinct(LogicalPlan plan, OperatorKey k, LogicalOperator input) {
- super(plan, k, rp);
+ super(plan, k);
+ mInput = input;
+ }
+
+ public LogicalOperator getInput() {
+ return mInput;
}
@Override
public Schema getSchema() throws FrontendException {
if (!mIsSchemaComputed && (null == mSchema)) {
// Get the schema of the parent
- Collection<LogicalOperator> s = mPlan.getSuccessors(this);
+ Collection<LogicalOperator> s = mPlan.getPredecessors(this);
try {
LogicalOperator op = s.iterator().next();
- if(null == op) {
+ if (null == op) {
+ log.info("getSchema: Operator not in plan");
throw new FrontendException("Could not find operator in plan");
}
mSchema = s.iterator().next().getSchema();
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LODivide.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LODivide extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LODivide.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LODivide(LogicalPlan plan, OperatorKey k, int rp,
+ public LODivide(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOEqual.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOEqual extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOEqual.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOEqual(LogicalPlan plan, OperatorKey k, int rp,
+ public LOEqual(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOFilter.java Thu May 8 14:25:22 2008
@@ -22,12 +22,15 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOFilter extends LogicalOperator {
private static final long serialVersionUID = 2L;
- private ExpressionOperator mCond;
private LogicalOperator mInput;
+ private LogicalPlan mComparisonPlan;
+ private static Log log = LogFactory.getLog(LOFilter.class);
/**
*
@@ -35,29 +38,26 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param cond
* the filter condition
* @param input
* the input that needs filtering
*/
- public LOFilter(LogicalPlan plan, OperatorKey k, int rp,
- ExpressionOperator cond, LogicalOperator input) {
- super(plan, k, rp);
- mCond = cond;
+ public LOFilter(LogicalPlan plan, OperatorKey k,
+ LogicalPlan comparisonPlan, LogicalOperator input) {
+ super(plan, k);
+ mComparisonPlan = comparisonPlan;
mInput = input;
- }
+ }
public LogicalOperator getInput() {
return mInput;
}
- public ExpressionOperator getCondition() {
- return mCond;
- }
+ public LogicalPlan getComparisonPlan() {
+ return mComparisonPlan;
+ }
@Override
public Schema getSchema() throws FrontendException {
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOForEach.java Thu May 8 14:25:22 2008
@@ -23,6 +23,9 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
public class LOForEach extends LogicalOperator {
@@ -34,36 +37,44 @@
* applied over the input.
*/
- private ArrayList<LogicalOperator> mOperators;
+ private LogicalPlan mForEachPlan;
+ private static Log log = LogFactory.getLog(LOForEach.class);
/**
* @param plan
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param operators
* the list of operators that are applied for each input
*/
- public LOForEach(LogicalPlan plan, OperatorKey k, int rp,
- ArrayList<LogicalOperator> operators) {
- super(plan, k, rp);
- mOperators = operators;
+ public LOForEach(LogicalPlan plan, OperatorKey k,
+ LogicalPlan foreachPlan) {
+
+ super(plan, k);
+ mForEachPlan = foreachPlan;
}
- public List<LogicalOperator> getOperators() {
- return mOperators;
+ public LogicalPlan getForEachPlan() {
+ return mForEachPlan;
}
@Override
public Schema getSchema() throws FrontendException {
- if (mSchema == null) {
+ if (!mIsSchemaComputed && (null == mSchema)) {
// Assuming that the last operator is the GENERATE
// foreach has to terminate with a GENERATE
- LogicalOperator last = mOperators.get(mOperators.size() - 1);
+ LogicalOperator last = null;
+ for(LogicalOperator op: mForEachPlan.getLeaves()) {
+ if(op instanceof LOGenerate) {
+ last = op;
+ break;
+ }
+ }
+ if(null == last) throw new FrontendException("Did not find generate in the foreach logicalplan");
+
+ log.debug("Last Operator: " + last.getClass().getName());
try {
mSchema = last.getSchema();
mIsSchemaComputed = true;
@@ -78,7 +89,7 @@
@Override
public String name() {
- return "Filter " + mKey.scope + "-" + mKey.id;
+ return "ForEach " + mKey.scope + "-" + mKey.id;
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGenerate.java Thu May 8 14:25:22 2008
@@ -19,12 +19,18 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Iterator;
+import java.util.Set;
+import java.util.Map;
+import java.util.HashMap;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOGenerate extends LogicalOperator {
private static final long serialVersionUID = 2L;
@@ -32,7 +38,10 @@
/**
* The projection list of this generate.
*/
- private ArrayList<ExpressionOperator> mProjections;
+ //private ArrayList<ExpressionOperator> mProjections;
+ private ArrayList<LogicalPlan> mGeneratePlans;
+ private ArrayList<Boolean> mFlatten;
+ private static Log log = LogFactory.getLog(LOGenerate.class);
/**
*
@@ -40,22 +49,45 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param projections
* the projection list of the generate
*/
- public LOGenerate(LogicalPlan plan, OperatorKey key, int rp,
- ArrayList<ExpressionOperator> projections) {
- super(plan, key, rp);
- mProjections = projections;
+
+ public LOGenerate(LogicalPlan plan, OperatorKey key,
+ ArrayList<LogicalPlan> generatePlans, ArrayList<Boolean> flatten) {
+ super(plan, key);
+ mGeneratePlans = generatePlans;
+ mFlatten = flatten;
+ }
+
+ /**
+ *
+ * @param plan
+ * Logical plan this operator is a part of.
+ * @param k
+ * Operator key to assign to this node.
+ * @param projection
+ * the projection of the generate
+ */
+
+ public LOGenerate(LogicalPlan plan, OperatorKey key,
+ LogicalPlan generatePlan, boolean flatten) {
+ super(plan, key);
+ mGeneratePlans = new ArrayList<LogicalPlan>();
+ mGeneratePlans.add(generatePlan);
+ mFlatten = new ArrayList<Boolean>();
+ mFlatten.add(flatten);
}
- public List<ExpressionOperator> getProjections() {
- return mProjections;
+
+ public List<LogicalPlan> getGeneratePlans() {
+ return mGeneratePlans;
}
+ public List<Boolean> getFlatten() {
+ return mFlatten;
+ }
+
@Override
public String name() {
return "Generate " + mKey.scope + "-" + mKey.id;
@@ -73,26 +105,95 @@
@Override
public Schema getSchema() throws FrontendException {
- if (mSchema == null) {
+ log.debug("Entering getSchema");
+ if (!mIsSchemaComputed && (null == mSchema)) {
List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(
- mProjections.size());
- for (ExpressionOperator op : mProjections) {
- String opAlias = op.getAlias();
- if (op.getType() == DataType.TUPLE) {
- try {
- fss.add(new Schema.FieldSchema(opAlias, op.getSchema()));
- } catch (FrontendException ioe) {
- mSchema = null;
- mIsSchemaComputed = false;
- throw ioe;
+ mGeneratePlans.size());
+
+ for (LogicalPlan plan : mGeneratePlans) {
+ log.debug("Number of roots in " + plan + " = " + plan.getRoots().size());
+ for(int i = 0; i < plan.getRoots().size(); ++i) {
+ log.debug("Root" + i + "= " + plan.getRoots().get(i));
+ }
+ LogicalOperator op = plan.getRoots().get(0);
+ log.debug("op: " + op.getClass().getName() + " " + op);
+ }
+ log.debug("Printed the roots of the generate plans");
+
+ Map<Schema.FieldSchema, String> flattenAlias = new HashMap<Schema.FieldSchema, String>();
+
+ for (int planCtr = 0; planCtr < mGeneratePlans.size(); ++planCtr) {
+ LogicalPlan plan = mGeneratePlans.get(planCtr);
+ LogicalOperator op = plan.getRoots().get(0);
+ log.debug("op: " + op.getClass().getName() + " " + op);
+ Set<String> aliases;
+ Iterator<String> iter = null;
+ String opSchemaAlias = null;
+ Schema s = null;
+
+ try {
+ s = op.getSchema();
+ if (null != s) {
+ log.debug("Printing aliases in LOGenerate");
+ s.printAliases();
+ aliases = op.getSchema().getAliases();
+ iter = aliases.iterator();
+ }
+ } catch (FrontendException ioe) {
+ mSchema = null;
+ mIsSchemaComputed = false;
+ throw ioe;
+ }
+
+ // ASSUMPTION
+ // Here I am assuming that the LOProject does not have multiple
+ // columns in the project, i.e., generate A.($1,$2,$3) as
+ // (name,age,gpa)
+
+ if ((null != iter) && (iter.hasNext())) {
+ opSchemaAlias = iter.next();
+ log.debug("iter.next: " + opSchemaAlias);
+ }
+
+
+ log.debug("Type: " + DataType.findTypeName(op.getType())
+ + " Alias: " + opSchemaAlias);
+
+ if (mFlatten.get(planCtr)) {
+ log.debug("Flatten");
+ for(Schema.FieldSchema fs: s.getFields()) {
+ Schema internalSchema = fs.schema;
+ Schema.FieldSchema newFs;
+ log.debug("Flatten level 1 internalSchema: " + internalSchema);
+ if(null != internalSchema) {
+ for(Schema.FieldSchema internalfs: internalSchema.getFields()) {
+ newFs = new Schema.FieldSchema(internalfs.alias, internalfs.schema);
+ fss.add(newFs);
+ log.debug("Flatten alias: " + opSchemaAlias+"::"+internalfs.alias);
+ flattenAlias.put(newFs, opSchemaAlias+"::"+internalfs.alias);
+ }
+ }
+ else {
+ newFs= new Schema.FieldSchema(fs.alias, fs.schema);
+ fss.add(newFs);
+ log.debug("Flatten alias: " + opSchemaAlias+"::"+fs.alias);
+ flattenAlias.put(newFs, opSchemaAlias+"::"+fs.alias);
+ }
}
} else {
- fss.add(new Schema.FieldSchema(opAlias, op.getType()));
+ fss.add(new Schema.FieldSchema(opSchemaAlias, s));
}
}
mSchema = new Schema(fss);
+ for(Schema.FieldSchema fs: mSchema.getFields()) {
+ String alias = flattenAlias.get(fs);
+ if(null != alias) {
+ mSchema.addAlias(alias, fs);
+ }
+ }
mIsSchemaComputed = true;
}
+ log.debug("Exiting getSchema");
return mSchema;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThan.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOGreaterThan extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOGreaterThan.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOGreaterThan(LogicalPlan plan, OperatorKey k, int rp,
+ public LOGreaterThan(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOGreaterThanEqual.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOGreaterThanEqual extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOGreaterThanEqual.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOGreaterThanEqual(LogicalPlan plan, OperatorKey k, int rp,
+ public LOGreaterThanEqual(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThan.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOLesserThan extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOLesserThan.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOLesserThan(LogicalPlan plan, OperatorKey k, int rp,
+ public LOLesserThan(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLesserThanEqual.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOLesserThanEqual extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOLesserThanEqual.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOLesserThanEqual(LogicalPlan plan, OperatorKey k, int rp,
+ public LOLesserThanEqual(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOLoad.java Thu May 8 14:25:22 2008
@@ -24,11 +24,14 @@
import java.util.Map;
import java.net.URL;
-import org.apache.pig.LoadFunc; // import org.apache.pig.impl.PigContext;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOLoad extends LogicalOperator {
private static final long serialVersionUID = 2L;
@@ -36,36 +39,34 @@
private FileSpec mInputFileSpec;
private LoadFunc mLoadFunc;
private URL mSchemaFile;
+ private static Log log = LogFactory.getLog(LOLoad.class);
/**
* @param plan
* LogicalPlan this operator is a part of.
* @param key
* OperatorKey for this operator
- * @param rp
- * Requested level of parallelism to be used in the sort.
* @param inputFileSpec
- * the file to be loaded
+ * the file to be loaded *
+ * @param schemaFile
+ * the file with the schema for the data to be loaded
+ *
*/
- public LOLoad(LogicalPlan plan, OperatorKey key, int rp,
- FileSpec inputFileSpec, URL schemaFile) throws IOException {
- super(plan, key, rp);
+ public LOLoad(LogicalPlan plan, OperatorKey key, FileSpec inputFileSpec,
+ URL schemaFile) throws IOException {
+ super(plan, key);
mInputFileSpec = inputFileSpec;
mSchemaFile = schemaFile;
- // TODO FIX
- // The code below is commented out as PigContext pulls in
- // HExecutionEngine which in turn is completely commented out
- // Also remove the commented out import org.apache.pig.impl.PigContext
-
- /*
- * try { mLoadFunc = (LoadFunc)
- * PigContext.instantiateFuncFromSpec(inputFileSpec.getFuncSpec()); }
- * catch (Exception e){ IOException ioe = new
- * IOException(e.getMessage()); ioe.setStackTrace(e.getStackTrace());
- * throw ioe; }
- */
+ try {
+ mLoadFunc = (LoadFunc)
+ PigContext.instantiateFuncFromSpec(inputFileSpec.getFuncSpec());
+ } catch (Exception e){
+ IOException ioe = new IOException(e.getMessage());
+ ioe.setStackTrace(e.getStackTrace());
+ throw ioe;
+ }
}
public FileSpec getInputFile() {
@@ -90,11 +91,17 @@
if (!mIsSchemaComputed && (null == mSchema)) {
// get the schema of the load function
try {
- mSchema = mLoadFunc.determineSchema(mSchemaFile);
+ //DEBUG
+ //System.out.println("Schema file: " + mSchema);
+ if(null != mSchemaFile) {
+ mSchema = mLoadFunc.determineSchema(mSchemaFile);
+ }
mIsSchemaComputed = true;
} catch (IOException ioe) {
FrontendException fee = new FrontendException(ioe.getMessage());
fee.initCause(ioe);
+ mIsSchemaComputed = false;
+ mSchema = null;
throw fee;
}
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMapLookup.java Thu May 8 14:25:22 2008
@@ -25,6 +25,8 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.plan.VisitorException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOMapLookup extends ExpressionOperator {
private static final long serialVersionUID = 2L;
@@ -35,8 +37,9 @@
*/
private ExpressionOperator mMap;
private Object mMapKey;
- private DataType mValueType;
+ private byte mValueType;
private Schema mValueSchema;
+ private static Log log = LogFactory.getLog(LOMapLookup.class);
/**
*
@@ -44,9 +47,6 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param map
* the map expression
* @param mapKey
@@ -56,10 +56,10 @@
* @param valueSchema
* schema of the value if the type is tuple
*/
- public LOMapLookup(LogicalPlan plan, OperatorKey key, int rp, ExpressionOperator map,
- Object mapKey, DataType valueType, Schema valueSchema)
+ public LOMapLookup(LogicalPlan plan, OperatorKey key, ExpressionOperator map,
+ Object mapKey, byte valueType, Schema valueSchema)
throws ParseException {
- super(plan, key, rp);
+ super(plan, key);
if (!DataType.isAtomic(mapKey)) {
throw new ParseException("Map key" + mapKey.toString()
@@ -79,7 +79,7 @@
return mMapKey;
}
- public DataType getValueType() {
+ public byte getValueType() {
return mValueType;
}
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMod.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOMod extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOMod.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOMod(LogicalPlan plan, OperatorKey k, int rp,
+ public LOMod(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOMultiply.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOMultiply extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOMultiply.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOMultiply(LogicalPlan plan, OperatorKey k, int rp,
+ public LOMultiply(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONegative.java Thu May 8 14:25:22 2008
@@ -32,15 +32,12 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param operand
* the only operand for a unary operator
*/
- public LONegative(LogicalPlan plan, OperatorKey k, int rp,
+ public LONegative(LogicalPlan plan, OperatorKey k,
ExpressionOperator operand) {
- super(plan, k, rp, operand);
+ super(plan, k, operand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONot.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LONot extends UnaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LONot.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param operand
* the only operand for a unary operator
*/
- public LONot(LogicalPlan plan, OperatorKey k, int rp,
+ public LONot(LogicalPlan plan, OperatorKey k,
ExpressionOperator operand) {
- super(plan, k, rp, operand);
+ super(plan, k, operand);
}
-
+
@Override
public Schema getSchema() {
return mSchema;
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LONotEqual.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LONotEqual extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LONotEqual.class);
/**
*
@@ -32,17 +35,14 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LONotEqual(LogicalPlan plan, OperatorKey k, int rp,
+ public LONotEqual(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
@Override
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOOr.java Thu May 8 14:25:22 2008
@@ -21,10 +21,13 @@
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOOr extends BinaryExpressionOperator {
private static final long serialVersionUID = 2L;
+ private static Log log = LogFactory.getLog(LOOr.class);
/**
*
@@ -32,19 +35,16 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param lhsOperand
* the left hand side operand
* @param rhsOperand
* the right hand side operand
*/
- public LOOr(LogicalPlan plan, OperatorKey k, int rp,
+ public LOOr(LogicalPlan plan, OperatorKey k,
ExpressionOperator lhsOperand, ExpressionOperator rhsOperand) {
- super(plan, k, rp, lhsOperand, rhsOperand);
+ super(plan, k, lhsOperand, rhsOperand);
}
-
+
@Override
public Schema getSchema() {
return mSchema;
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java?rev=654629&r1=654628&r2=654629&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOProject.java Thu May 8 14:25:22 2008
@@ -21,19 +21,47 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+/**
+ * LOProject is designed like a singly linked list; A few examples will
+ * illustrate the point about the linked list nature of the design;
+ * a = load 'input1' as (name, age);
+ * b = group a by name;
+ * foreach b generate a, a.name;
+ * The project operator occurs in two places in the above script:
+ * generate a(here) and a.name(here)
+ * In the first occurrence, we are trying to project the elements of
+ * the bag a; In order to retrieve the bag, we need to project the
+ * the second column ($1) or column number 1 (using the zero based index)
+ * from the input (the relation or bag b)
+ * In the second occurence, we are trying to project the first column
+ * ($0) or column number 0 from the bag a which in turn is the column
+ * number 1 in the relation b; As you can see, the nested structure or
+ * the singly linked list nature is clearly visible;
+ * Given that it's a singly linked list, the null pointer or the sentinel
+ * is marked explictly using the boolean variable mSentinel; The sentinel
+ * is marked true only when the input is a relational operator; This occurs
+ * when we create the innermost operator
+ */
public class LOProject extends ExpressionOperator {
private static final long serialVersionUID = 2L;
/**
* The expression and the column to be projected.
*/
- private ExpressionOperator mExp;
- private List<String> mProjection;
+ private LogicalOperator mExp;
+ private List<Integer> mProjection;
+ private boolean mIsStar = false;
+ private static Log log = LogFactory.getLog(LOProject.class);
+ private boolean mSentinel;
/**
*
@@ -41,19 +69,21 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param exp
* the expression which might contain the column to project
* @param projection
* the list of columns to project
*/
- public LOProject(LogicalPlan plan, OperatorKey key, int rp,
- ExpressionOperator exp, List<String> projection) {
- super(plan, key, rp);
+ public LOProject(LogicalPlan plan, OperatorKey key, LogicalOperator exp,
+ List<Integer> projection) {
+ super(plan, key);
mExp = exp;
mProjection = projection;
+ if(mExp instanceof ExpressionOperator) {
+ mSentinel = false;
+ } else {
+ mSentinel = true;
+ }
}
/**
@@ -62,30 +92,53 @@
* Logical plan this operator is a part of.
* @param k
* Operator key to assign to this node.
- * @param rp
- * degree of requested parallelism with which to execute this
- * node.
* @param exp
* the expression which might contain the column to project
* @param projection
* the column to project
*/
- public LOProject(LogicalPlan plan, OperatorKey key, int rp,
- ExpressionOperator exp, String projection) {
- super(plan, key, rp);
+ public LOProject(LogicalPlan plan, OperatorKey key, LogicalOperator exp,
+ Integer projection) {
+ super(plan, key);
mExp = exp;
- mProjection = new ArrayList<String>(1);
+ mProjection = new ArrayList<Integer>(1);
mProjection.add(projection);
+ if(mExp instanceof ExpressionOperator) {
+ mSentinel = false;
+ } else {
+ mSentinel = true;
+ }
}
- public ExpressionOperator getExpression() {
+ public LogicalOperator getExpression() {
return mExp;
}
- public List<String> getProjection() {
+ public List<Integer> getProjection() {
return mProjection;
}
+ public int getCol() {
+ if (mProjection.size() != 1)
+ throw new RuntimeException(
+ "Internal error: improper use of getCol in "
+ + LOProject.class.getName());
+ return mProjection.get(0);
+
+ }
+
+ public void setStar(boolean b) {
+ mIsStar = b;
+ }
+
+ public boolean getSentinel() {
+ return mSentinel;
+ }
+
+ public void setSentinel(boolean b) {
+ mSentinel = b;
+ }
+
@Override
public String name() {
return "Project " + mKey.scope + "-" + mKey.id;
@@ -97,7 +150,159 @@
}
@Override
- public Schema getSchema() {
+ public Schema getSchema() throws FrontendException {
+ log.debug("Inside getSchema");
+ log.debug("Number of columns: " + mProjection.size());
+ for (int i : mProjection) {
+ log.debug("Column: " + i);
+ }
+ LogicalOperator expressionOperator = mExp;
+ log.debug("expressionOperator = " + expressionOperator);
+ log.debug("mIsStar: " + mIsStar);
+
+ if (!mIsSchemaComputed && (null == mSchema)) {
+
+ if (mIsStar) {
+ log.debug("mIsStar is true");
+ try {
+ if (null != expressionOperator) {
+ log.debug("expressionOperator is not null "
+ + expressionOperator.getClass().getName() + " " + expressionOperator);
+ mSchema = expressionOperator.getSchema();
+ }
+ mIsSchemaComputed = true;
+ } catch (FrontendException ioe) {
+ mSchema = null;
+ mIsSchemaComputed = false;
+ throw ioe;
+ }
+ log.debug("mIsStar is true, returning schema of expressionOperator");
+ log.debug("Exiting getSchema()");
+ return mSchema;
+ } else {
+ List<Schema.FieldSchema> fss = new ArrayList<Schema.FieldSchema>(
+ mProjection.size());
+
+ if (mProjection.size() == 1) {
+ log.debug("Only one element");
+ try {
+ Schema s = expressionOperator.getSchema();
+ if (null != s) {
+ log.debug("Getting the fieldschema for column: " + mProjection.get(0));
+ Schema.FieldSchema fs = null;
+ if(!mSentinel) {
+ log.debug("We are in the outer part of the projection");
+ fs = s.getField(0);
+ log.debug("fs in outer part: " + fs);
+ s = fs.schema;
+ log.debug("s: " + s);
+ if(null != s) {
+ fs = s.getField(mProjection.get(0));
+ log.debug("fs in outer part after unwrapping: " + fs);
+ } else {
+ fs = null;
+ log.debug("fs in outer part after unwrapping: " + fs);
+ }
+ } else {
+ log.debug("We are in the inner most part of the projection");
+ fs = s.getField(mProjection.get(0));
+ }
+ if (null != fs) {
+ log.debug("fs.type: "
+ + DataType.findTypeName(fs.type)
+ + " fs.schema: " + fs.schema);
+ if (fs.type == DataType.BAG
+ || fs.type == DataType.TUPLE) {
+ if (null != fs.schema) {
+ log.debug("fs.schema aliases");
+ fs.schema.printAliases();
+ }
+
+ //mSchema = fs.schema;
+ mSchema = new Schema(new Schema.FieldSchema(fs.alias, fs.schema));
+ //mSchema = new Schema(fs);
+ } else {
+ mSchema = new Schema(fs);
+ }
+ } else {
+ log.debug("expressionOperator.schema is null");
+ mSchema = new Schema(new Schema.FieldSchema(null,
+ DataType.BYTEARRAY));
+ }
+ } else {
+ log.debug("expressionOperator.schema is null");
+ mSchema = new Schema(new Schema.FieldSchema(null,
+ DataType.BYTEARRAY));
+ }
+ } catch (Exception e) {
+ mSchema = null;
+ mIsSchemaComputed = false;
+ throw new FrontendException(e.getMessage());
+ }
+ mIsSchemaComputed = true;
+ log.debug("Exiting getSchema");
+ return mSchema;
+ }
+
+ if (null != expressionOperator) {
+ log.debug("expressionOperator is not null");
+
+ Schema s = expressionOperator.getSchema();
+ log.debug("s: " + s);
+ for (int colNum : mProjection) {
+ log.debug("Col: " + colNum);
+ if (null != s) {
+ try {
+ Schema.FieldSchema tempFs = s.getField(0);
+ if (null != tempFs) {
+ if(null != tempFs.schema) {
+ Schema.FieldSchema fs = tempFs.schema.getField(colNum);
+ log.debug("fs.type: "
+ + DataType.findTypeName(fs.type)
+ + " fs.schema: " + fs.schema);
+ if(fs.type == DataType.BAG || fs.type == DataType.TUPLE) {
+ if (null != fs.schema) {
+ log.debug("fs.schema aliases");
+ fs.schema.printAliases();
+ }
+ fss.add(new Schema.FieldSchema(fs.alias, fs.schema));
+ } else {
+ fss.add(fs);
+ }
+ } else {
+ log.debug("expressionOperator.fs.schema is null");
+ fss.add(new Schema.FieldSchema(null,
+ DataType.BYTEARRAY));
+ }
+ } else {
+ log.debug("Could not find column " + colNum+ " in schema");
+ throw new FrontendException(
+ "Could not find column " + colNum
+ + " in schema");
+ }
+ } catch (Exception e) {
+ log.debug("Caught exception: " + e.getMessage());
+ mSchema = null;
+ mIsSchemaComputed = false;
+ throw new FrontendException(e.getMessage());
+ }
+ } else {
+ log.debug("expressionOperator.schema is null");
+ fss.add(new Schema.FieldSchema(null,
+ DataType.BYTEARRAY));
+ }
+ }
+ } else {
+ log.debug("expressionOperator is null");
+ fss.add(new Schema.FieldSchema(null, DataType.BYTEARRAY));
+ }
+ mSchema = new Schema(fss);
+ mIsSchemaComputed = true;
+ log.debug("mIsStar is false, returning computed schema of expressionOperator");
+ }
+ }
+
+ log.debug("Exiting getSchema");
return mSchema;
}