You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/03/15 04:28:28 UTC
svn commit: r923043 [1/5] - in /hadoop/pig/trunk:
src/org/apache/pig/backend/hadoop/executionengine/
src/org/apache/pig/experimental/logical/
src/org/apache/pig/experimental/logical/expression/
src/org/apache/pig/experimental/logical/optimizer/ src/org...
Author: daijy
Date: Mon Mar 15 03:28:27 2010
New Revision: 923043
URL: http://svn.apache.org/viewvc?rev=923043&view=rev
Log:
PIG-1178:LogicalPlan and Optimizer are too complex and hard to work with (pig_1178_3.3.patch)
Added:
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BagDereferenceExpression.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BinCondExpression.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/RegexExpression.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/UserFuncExpression.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOCogroup.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOSplit.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOSplitOutput.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOUnion.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/SchemaNotDefinedException.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/AddForEach.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/ColumnMapKeyPrune.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/ColumnPruneHelper.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/MapKeysPruneHelper.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/WholePlanRule.java
hadoop/pig/trunk/test/org/apache/pig/test/TestExperimentalColumnPrune.java
hadoop/pig/trunk/test/org/apache/pig/test/TestExperimentalPruneMapKeys.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumnNewLogicalPlan.java
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/LogicalPlanMigrationVistor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ExpToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/LogicalExpressionVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ProjectExpression.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllExpressionVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllSameVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/LogicalPlanOptimizer.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/PlanPrinter.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/ProjectionPatcher.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/SchemaPatcher.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/UidStamper.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOGenerate.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOInnerLoad.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LOLoad.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LogToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LogicalPlan.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LogicalPlanVisitor.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LogicalRelationalOperator.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/relational/LogicalSchema.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/FilterAboveForeach.java
hadoop/pig/trunk/src/org/apache/pig/experimental/logical/rules/PushUpFilter.java
hadoop/pig/trunk/src/org/apache/pig/experimental/plan/BaseOperatorPlan.java
hadoop/pig/trunk/test/org/apache/pig/test/TestExperimentalLogToPhyTranslationVisitor.java
hadoop/pig/trunk/test/org/apache/pig/test/TestLogicalPlanMigrationVisitor.java
hadoop/pig/trunk/test/org/apache/pig/test/Util.java
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Mon Mar 15 03:28:27 2010
@@ -18,6 +18,7 @@
package org.apache.pig.backend.hadoop.executionengine;
+import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
@@ -55,6 +56,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
import org.apache.pig.experimental.logical.LogicalPlanMigrationVistor;
+import org.apache.pig.experimental.logical.optimizer.PlanPrinter;
import org.apache.pig.experimental.logical.optimizer.UidStamper;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileSpec;
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/LogicalPlanMigrationVistor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/LogicalPlanMigrationVistor.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/LogicalPlanMigrationVistor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/LogicalPlanMigrationVistor.java Mon Mar 15 03:28:27 2010
@@ -19,12 +19,13 @@ package org.apache.pig.experimental.logi
import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Iterator;
import java.util.List;
import org.apache.pig.data.DataType;
import org.apache.pig.experimental.logical.expression.AddExpression;
import org.apache.pig.experimental.logical.expression.AndExpression;
+import org.apache.pig.experimental.logical.expression.BagDereferenceExpression;
+import org.apache.pig.experimental.logical.expression.BinCondExpression;
import org.apache.pig.experimental.logical.expression.CastExpression;
import org.apache.pig.experimental.logical.expression.ConstantExpression;
import org.apache.pig.experimental.logical.expression.DivideExpression;
@@ -44,10 +45,13 @@ import org.apache.pig.experimental.logic
import org.apache.pig.experimental.logical.expression.NotExpression;
import org.apache.pig.experimental.logical.expression.OrExpression;
import org.apache.pig.experimental.logical.expression.ProjectExpression;
+import org.apache.pig.experimental.logical.expression.RegexExpression;
import org.apache.pig.experimental.logical.expression.SubtractExpression;
+import org.apache.pig.experimental.logical.expression.UserFuncExpression;
import org.apache.pig.experimental.logical.relational.LOInnerLoad;
import org.apache.pig.experimental.logical.relational.LogicalRelationalOperator;
import org.apache.pig.experimental.logical.relational.LogicalSchema;
+import org.apache.pig.experimental.plan.Operator;
import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.logicalLayer.ExpressionOperator;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -92,6 +96,7 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.impl.logicalLayer.LOVisitor;
import org.apache.pig.impl.logicalLayer.LogicalOperator;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
+import org.apache.pig.impl.logicalLayer.LOCogroup.GROUPTYPE;
import org.apache.pig.impl.logicalLayer.LOJoin.JOINTYPE;
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema;
@@ -100,6 +105,9 @@ import org.apache.pig.impl.plan.PlanWalk
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.util.MultiMap;
+/**
+ * Translate old logical plan into new logical plan
+ */
public class LogicalPlanMigrationVistor extends LOVisitor {
private org.apache.pig.experimental.logical.relational.LogicalPlan logicalPlan;
private HashMap<LogicalOperator, LogicalRelationalOperator> opsMap;
@@ -153,7 +161,39 @@ public class LogicalPlanMigrationVistor
}
public void visit(LOCogroup cg) throws VisitorException {
- throw new VisitorException("LOCogroup is not supported.");
+
+ // Get the GroupType information
+ org.apache.pig.experimental.logical.relational.LOCogroup.GROUPTYPE grouptype;
+ if( cg.getGroupType() == GROUPTYPE.COLLECTED ) {
+ grouptype = org.apache.pig.experimental.logical.relational.LOCogroup.GROUPTYPE.COLLECTED;
+ } else {
+ grouptype = org.apache.pig.experimental.logical.relational.LOCogroup.GROUPTYPE.REGULAR;
+ }
+
+ // Convert the multimap of expressionplans to a new way
+ ArrayList<LogicalOperator> inputs = (ArrayList<LogicalOperator>) cg.getInputs();
+ MultiMap<Integer, LogicalExpressionPlan> newExpressionPlans =
+ new MultiMap<Integer, LogicalExpressionPlan>();
+
+ for( int i = 0; i < inputs.size(); i++ ) {
+ ArrayList<LogicalPlan> plans =
+ (ArrayList<LogicalPlan>) cg.getGroupByPlans().get(inputs.get(i));
+ for( LogicalPlan plan : plans ) {
+ LogicalExpressionPlan expPlan = translateExpressionPlan(plan);
+ newExpressionPlans.put(Integer.valueOf(i), expPlan);
+ }
+ }
+
+ org.apache.pig.experimental.logical.relational.LOCogroup newCogroup =
+ new org.apache.pig.experimental.logical.relational.LOCogroup
+ (logicalPlan, newExpressionPlans, grouptype, cg.getInner(),
+ cg.getRequestedParallelism() );
+
+ newCogroup.setAlias(cg.getAlias());
+
+ logicalPlan.add(newCogroup);
+ opsMap.put(cg, newCogroup);
+ translateConnection(cg, newCogroup);
}
public void visit(LOJoin loj) throws VisitorException {
@@ -220,51 +260,10 @@ public class LogicalPlanMigrationVistor
innerPlan.add(gen);
List<LogicalPlan> ll = forEach.getForEachPlans();
- int index = 0;
for(int i=0; i<ll.size(); i++) {
LogicalPlan lp = ll.get(i);
ForeachInnerPlanVisitor v = new ForeachInnerPlanVisitor(newForeach, forEach, lp);
- v.visit();
-
- // get the logical plan for this inner plan and merge it as subplan of LOGenerator
- if (v.tmpPlan.size() > 0) {
- // add all operators into innerplan
- Iterator<org.apache.pig.experimental.plan.Operator> iter = v.tmpPlan.getOperators();
- while(iter.hasNext()) {
- innerPlan.add(iter.next());
- }
-
- // connect sinks to generator
- // the list returned may not be in correct order, so check annotation
- // to guarantee correct order
- List<org.apache.pig.experimental.plan.Operator> s = v.tmpPlan.getSinks();
- for(int j=0; j<s.size(); j++) {
- for(org.apache.pig.experimental.plan.Operator op: s) {
- if (Integer.valueOf(j+index).equals(op.getAnnotation("inputNo"))) {
- innerPlan.connect(op, gen);
- break;
- }
- }
- }
- index += s.size();
-
- // copy connections
- iter = v.tmpPlan.getOperators();
- while(iter.hasNext()) {
- org.apache.pig.experimental.plan.Operator op = iter.next();
- op.removeAnnotation("inputNo");
- try{
- List<org.apache.pig.experimental.plan.Operator> succ = v.tmpPlan.getSuccessors(op);
- if (succ != null) {
- for(org.apache.pig.experimental.plan.Operator ss: succ) {
- innerPlan.connect(op, ss);
- }
- }
- }catch(Exception e) {
- throw new VisitorException(e);
- }
- }
- }
+ v.visit();
expPlans.add(v.exprPlan);
}
@@ -305,10 +304,17 @@ public class LogicalPlanMigrationVistor
}
public void visit(LOSplit split) throws VisitorException {
- throw new VisitorException("LOSplit is not supported.");
+ org.apache.pig.experimental.logical.relational.LOSplit newSplit =
+ new org.apache.pig.experimental.logical.relational.LOSplit(logicalPlan);
+
+ newSplit.setAlias(split.getAlias());
+ newSplit.setRequestedParallelism(split.getRequestedParallelism());
+
+ logicalPlan.add(newSplit);
+ opsMap.put(split, newSplit);
+ translateConnection(split, newSplit);
}
-
public void visit(LOGenerate g) throws VisitorException {
throw new VisitorException("LOGenerate is not supported.");
}
@@ -347,12 +353,31 @@ public class LogicalPlanMigrationVistor
translateConnection(store, newStore);
}
- public void visit(LOUnion u) throws VisitorException {
- throw new VisitorException("LOUnion is not supported.");
- }
-
- public void visit(LOSplitOutput sop) throws VisitorException {
- throw new VisitorException("LOSplitOutput is not supported.");
+ public void visit(LOUnion union) throws VisitorException {
+ org.apache.pig.experimental.logical.relational.LOUnion newUnion =
+ new org.apache.pig.experimental.logical.relational.LOUnion(logicalPlan);
+
+ newUnion.setAlias(union.getAlias());
+ newUnion.setRequestedParallelism(union.getRequestedParallelism());
+ logicalPlan.add(newUnion);
+ opsMap.put(union, newUnion);
+ translateConnection(union, newUnion);
+ }
+
+ public void visit(LOSplitOutput splitOutput) throws VisitorException {
+ org.apache.pig.experimental.logical.relational.LOSplitOutput newSplitOutput =
+ new org.apache.pig.experimental.logical.relational.LOSplitOutput(logicalPlan);
+
+ LogicalPlan filterPlan = splitOutput.getConditionPlan();
+ LogicalExpressionPlan newFilterPlan = translateExpressionPlan(filterPlan);
+
+ newSplitOutput.setFilterPlan(newFilterPlan);
+ newSplitOutput.setAlias(splitOutput.getAlias());
+ newSplitOutput.setRequestedParallelism(splitOutput.getRequestedParallelism());
+
+ logicalPlan.add(newSplitOutput);
+ opsMap.put(splitOutput, newSplitOutput);
+ translateConnection(splitOutput, newSplitOutput);
}
public void visit(LODistinct dt) throws VisitorException {
@@ -363,6 +388,7 @@ public class LogicalPlanMigrationVistor
throw new VisitorException("LOCross is not supported.");
}
+ // visitor to translate expressions
public class LogicalExpPlanMigrationVistor extends LOVisitor {
protected org.apache.pig.experimental.logical.expression.LogicalExpressionPlan exprPlan;
@@ -393,21 +419,14 @@ public class LogicalPlanMigrationVistor
public void visit(LOProject project) throws VisitorException {
int col = project.getCol();
+
LogicalOperator lg = project.getExpression();
LogicalOperator succed = oldLogicalPlan.getSuccessors(lg).get(0);
int input = oldLogicalPlan.getPredecessors(succed).indexOf(lg);
// get data type of projection
- byte t = DataType.BYTEARRAY;
- try {
- Schema s = lg.getSchema();
- if (s != null) {
- t = s.getField(col).type;
- }
- }catch(Exception e) {
- throw new VisitorException(e);
- }
- ProjectExpression pe = new ProjectExpression(exprPlan, t, input, col);
+ byte t = project.getType();
+ ProjectExpression pe = new ProjectExpression(exprPlan, t, input, project.isStar()?-1:col);
exprPlan.add(pe);
exprOpsMap.put(project, pe);
@@ -466,12 +485,27 @@ public class LogicalPlanMigrationVistor
exprOpsMap.put(op, eq);
}
- public void visit(LOUserFunc func) throws VisitorException {
- throw new VisitorException("LOUserFunc is not supported.");
+ public void visit(LOUserFunc op) throws VisitorException {
+ UserFuncExpression exp = new UserFuncExpression(exprPlan, op.getFuncSpec(), op.getType());
+
+ List<ExpressionOperator> args = op.getArguments();
+
+ for( ExpressionOperator arg : args ) {
+ LogicalExpression expArg = exprOpsMap.get(arg);
+ exprPlan.connect(exp, expArg);
+ }
+
+ exprOpsMap.put(op, exp);
}
- public void visit(LOBinCond binCond) throws VisitorException {
- throw new VisitorException("LOBinCond is not supported.");
+ public void visit(LOBinCond op) throws VisitorException {
+ ExpressionOperator condition = op.getCond();
+ ExpressionOperator left = op.getLhsOp();
+ ExpressionOperator right = op.getRhsOp();
+
+ BinCondExpression exp = new BinCondExpression(exprPlan, op.getType(),
+ exprOpsMap.get(condition), exprOpsMap.get(left), exprOpsMap.get(right));
+ exprOpsMap.put(op, exp);
}
public void visit(LOCast cast) throws VisitorException {
@@ -483,8 +517,13 @@ public class LogicalPlanMigrationVistor
exprOpsMap.put(cast, c);
}
- public void visit(LORegexp regexp) throws VisitorException {
- throw new VisitorException("LORegexp is not supported.");
+ public void visit(LORegexp binOp) throws VisitorException {
+ ExpressionOperator left = binOp.getLhsOperand();
+ ExpressionOperator right = binOp.getRhsOperand();
+
+ RegexExpression ae = new RegexExpression(exprPlan, binOp.getType()
+ , exprOpsMap.get(left), exprOpsMap.get(right));
+ exprOpsMap.put(binOp, ae);
}
public void visit(LONotEqual op) throws VisitorException {
@@ -502,7 +541,7 @@ public class LogicalPlanMigrationVistor
AddExpression ae = new AddExpression(exprPlan, binOp.getType()
, exprOpsMap.get(left), exprOpsMap.get(right));
- exprOpsMap.put(binOp, ae);
+ exprOpsMap.put(binOp, ae);
}
public void visit(LOSubtract binOp) throws VisitorException {
@@ -600,19 +639,23 @@ public class LogicalPlanMigrationVistor
}
// visitor to translate the inner plan of foreach
- // it has all the operators allowed in the inner plan of foreach
+ // it contains methods to translate all the operators that are allowed
+ // in the inner plan of foreach
public class ForeachInnerPlanVisitor extends LogicalExpPlanMigrationVistor {
private org.apache.pig.experimental.logical.relational.LOForEach foreach;
+ private org.apache.pig.experimental.logical.relational.LogicalPlan newInnerPlan;
private LOForEach oldForeach;
+ private org.apache.pig.experimental.plan.Operator gen;
private int inputNo;
private HashMap<LogicalOperator, LogicalRelationalOperator> innerOpsMap;
- private org.apache.pig.experimental.logical.relational.LogicalPlan tmpPlan;
public ForeachInnerPlanVisitor(org.apache.pig.experimental.logical.relational.LOForEach foreach, LOForEach oldForeach, LogicalPlan plan) {
super(plan);
this.foreach = foreach;
- org.apache.pig.experimental.logical.relational.LogicalPlan newInnerPlan = foreach.getInnerPlan();
- org.apache.pig.experimental.plan.Operator gen = newInnerPlan.getSinks().get(0);
+ newInnerPlan = foreach.getInnerPlan();
+
+ // get next inputNo
+ gen = newInnerPlan.getSinks().get(0);
try {
inputNo = 0;
List<org.apache.pig.experimental.plan.Operator> suc = newInnerPlan.getPredecessors(gen);
@@ -624,8 +667,7 @@ public class LogicalPlanMigrationVistor
}
this.oldForeach = oldForeach;
- innerOpsMap = new HashMap<LogicalOperator, LogicalRelationalOperator>();
- tmpPlan = new org.apache.pig.experimental.logical.relational.LogicalPlan();
+ innerOpsMap = new HashMap<LogicalOperator, LogicalRelationalOperator>();
}
public void visit(LOProject project) throws VisitorException {
@@ -635,20 +677,48 @@ public class LogicalPlanMigrationVistor
// if this projection is to get a field from outer plan, change it
// to LOInnerLoad
- LOInnerLoad innerLoad = new LOInnerLoad(tmpPlan, foreach, project.getCol());
- // mark the input index of this subtree under LOGenerate
- // the successors of innerLoad should also annotatet the same inputNo
- innerLoad.annotate("inputNo", Integer.valueOf(inputNo));
- tmpPlan.add(innerLoad);
+ LOInnerLoad innerLoad = new LOInnerLoad(newInnerPlan, foreach, project.isStar()?-1:project.getCol());
+
+ newInnerPlan.add(innerLoad);
innerOpsMap.put(project, innerLoad);
- }
- List<LogicalOperator> ll = mPlan.getSuccessors(project);
- if (ll == null || ll.get(0) instanceof ExpressionOperator) {
- ProjectExpression pe = new ProjectExpression(exprPlan, project.getType(), inputNo++, 0);
- exprPlan.add(pe);
- exprOpsMap.put(project, pe);
- translateConnection(project, pe);
+
+ LogicalOperator succ = null;
+ if (mPlan.getSuccessors(project) != null) {
+ succ = mPlan.getSuccessors(project).get(0);
+ }
+
+ // The logical plan part for this foreach plan is done, add ProjectExpression
+ // into expression plan.
+ if (succ == null || succ instanceof ExpressionOperator) {
+
+ // The logical plan part is done, add this sub plan under LOGenerate,
+ // and prepare for the expression plan
+ newInnerPlan.connect(innerLoad, gen);
+
+ List<LogicalOperator> ll = mPlan.getSuccessors(project);
+ if (ll == null || ll.get(0) instanceof ExpressionOperator) {
+ ProjectExpression pe = new ProjectExpression(exprPlan, project.getType(), inputNo++, project.isStar()?-1:0);
+ exprPlan.add(pe);
+ exprOpsMap.put(project, pe);
+ translateConnection(project, pe);
+ }
+ }
+ }
+
+ // This case occurs when there are two projects one after another
+ // These projects in combination project a column (bag) out of a tuple
+ // and then project a column out of this projected bag
+ // Here we merge these two projects into one BagDereferenceExpression
+ if( op instanceof LOProject ) {
+ ProjectExpression projectExp = (ProjectExpression) exprOpsMap.get(op);
+
+ // Add the bag in the plan
+ BagDereferenceExpression bagDR = new BagDereferenceExpression(
+ exprPlan, project.getType(), project.getProjection(), projectExp);
+ exprOpsMap.put(project, bagDR);
+
+ translateConnection( project, bagDR );
}
}
Added: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BagDereferenceExpression.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BagDereferenceExpression.java?rev=923043&view=auto
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BagDereferenceExpression.java (added)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BagDereferenceExpression.java Mon Mar 15 03:28:27 2010
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.experimental.logical.expression;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.experimental.logical.relational.LogicalRelationalOperator;
+import org.apache.pig.experimental.logical.relational.LogicalSchema;
+import org.apache.pig.experimental.logical.relational.LogicalSchema.LogicalFieldSchema;
+import org.apache.pig.experimental.plan.Operator;
+import org.apache.pig.experimental.plan.OperatorPlan;
+import org.apache.pig.experimental.plan.PlanVisitor;
+
+/**
+ *
+ * This is a special case Expression and violates some of the rules of an
+ * Expression.
+ * Violation:
+ * It has multiple Uids ( though not tracked there can be multiple uids
+ * for this expression )
+ *
+ * This is a special operator which handles the case described below
+ * Tuple( a:int, b:bag{ b_a:int, b_b:float } ) -->
+ * BagDereference ( 0 ) --> project( 0, 1 ) --> bag{ b_a:float }
+ *
+ *
+ *
+ * i.e. First input ( 0 ), second column ( 1 ) and first column of the bag.
+ *
+ */
+public class BagDereferenceExpression extends ColumnExpression {
+
+ private List<Integer> columns;// The column in the input bag which the project references.
+ // Count is zero based.
+
+ public BagDereferenceExpression(OperatorPlan plan, byte type, int colNum, ProjectExpression exp) {
+ super( "BagDereference", plan, type );
+ plan.add(this);
+ plan.connect(this, exp);
+ columns = new ArrayList<Integer>();
+ columns.add(colNum);
+ }
+
+ public BagDereferenceExpression(OperatorPlan plan, byte type, List<Integer> columnNums, ProjectExpression exp) {
+ super( "BagDereference", plan, type );
+ plan.add(this);
+ plan.connect(this, exp);
+ columns = new ArrayList<Integer>();
+ columns.addAll(columnNums);
+ }
+
+ /**
+ * @link org.apache.pig.experimental.plan.Operator#accept(org.apache.pig.experimental.plan.PlanVisitor)
+ */
+ @Override
+ public void accept(PlanVisitor v) throws IOException {
+ if (!(v instanceof LogicalExpressionVisitor)) {
+ throw new IOException("Expected LogicalExpressionVisitor");
+ }
+ ((LogicalExpressionVisitor)v).visitBagDereference(this);
+ }
+
+ /**
+ * Column number in the bag at column <code>col</code>
+ * this Dereference References. The column number is the column
+ * in the relational operator that contains this expression. The count
+ * is zero based.
+ * @return column number
+ */
+ public int getBagColNum() {
+ if( columns.size() != 1 ) {
+ throw new RuntimeException(
+ "Internal error: improper use of getBagColNum in "
+ + BagDereferenceExpression.class.getName());
+ }
+ return columns.get(0);
+ }
+
+ public List<Integer> getBagColumns() {
+ return columns;
+ }
+
+ public void setBagColumns(List<Integer> columns) {
+ this.columns.clear();
+ this.columns.addAll(columns);
+ }
+
+ public void setBagColNum(int colNum) {
+ columns.clear();
+ columns.add(colNum);
+ }
+
+ /**
+ * Set the type of the projection.
+ * @param type to set this projection to
+ */
+ public void setType(byte type) {
+ this.type = type;
+ }
+
+ @Override
+ public void setUid(LogicalRelationalOperator currentOp) throws IOException {
+
+ // We generate a new Uid so it is easy to compute schema
+ uid = getNextUid();
+// ProjectExpression prj = getProjectExpression();
+// LogicalRelationalOperator referent = prj.findReferent(currentOp);
+//
+// LogicalSchema schema = referent.getSchema();
+// if (schema != null) {
+// LogicalFieldSchema field = schema.getField(prj.getColNum());
+// if( field.schema == null || field.schema.size() == 0 ||
+// ( field.schema.getField(0).type == DataType.TUPLE &&
+// field.schema.getField(0).schema == null ) ) {
+// throw new IOException("No schema found in bag for BagDereference!");
+// }
+// if( field.schema.getField(0).type == DataType.TUPLE ) {
+// uid = field.schema.getField(0).schema.getField(columns.get(0)).uid;
+// } else {
+// uid = field.schema.getField(columns.get(0)).uid;
+// }
+// } else {
+// // If the schema of referent is null, we kindof create a uid so we
+// // can track it in remaining plan
+// uid = getNextUid();
+// }
+ }
+
+ @Override
+ public boolean isEqual(Operator other) {
+ if (other != null && other instanceof BagDereferenceExpression) {
+ BagDereferenceExpression po = (BagDereferenceExpression)other;
+ try {
+ if( po.columns.size() != columns.size() ) {
+ return false;
+ }
+ return po.columns.containsAll(columns) && getProjectExpression().isEqual(po.getProjectExpression());
+ } catch (IOException e) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ public ProjectExpression getProjectExpression() throws IOException {
+ if( plan.getSuccessors(this).size() < 1 ) {
+ throw new IOException("Could not find a related project Expression for BagDereference");
+ }
+ return (ProjectExpression) plan.getSuccessors(this).get(0);
+ }
+
+ public String toString() {
+ StringBuilder msg = new StringBuilder();
+
+ msg.append("(Name: " + name + " Type: " + type + " Uid: " + uid +
+ " Column: " + columns + ")");
+
+ return msg.toString();
+ }
+}
Added: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BinCondExpression.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BinCondExpression.java?rev=923043&view=auto
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BinCondExpression.java (added)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/BinCondExpression.java Mon Mar 15 03:28:27 2010
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.experimental.logical.expression;
+
+import java.io.IOException;
+
+import org.apache.pig.experimental.plan.Operator;
+import org.apache.pig.experimental.plan.OperatorPlan;
+import org.apache.pig.experimental.plan.PlanVisitor;
+
+public class BinCondExpression extends LogicalExpression {
+
+ /**
+ * Will add this operator to the plan and connect it to the
+ * left and right hand side operators and the condition operator
+ * @param plan plan this operator is part of
+ * @param b Datatype of this expression
+ * @param lhs expression on its left hand side
+ * @param rhs expression on its right hand side
+ */
+ public BinCondExpression(OperatorPlan plan,
+ byte b,
+ LogicalExpression condition,
+ LogicalExpression lhs,
+ LogicalExpression rhs) {
+ super("BinCond", plan, b);
+ plan.add(this);
+ plan.connect(this, condition);
+ plan.connect(this, lhs);
+ plan.connect(this, rhs);
+ }
+
+ /**
+ * Returns the operator which handles this condition
+ * @return expression which handles the condition
+ * @throws IOException
+ */
+ public LogicalExpression getCondition() throws IOException {
+ return (LogicalExpression)plan.getSuccessors(this).get(0);
+ }
+
+ /**
+ * Get the left hand side of this expression.
+ * @return expression on the left hand side
+ * @throws IOException
+ */
+ public LogicalExpression getLhs() throws IOException {
+ return (LogicalExpression)plan.getSuccessors(this).get(1);
+ }
+
+ /**
+ * Get the right hand side of this expression.
+ * @return expression on the right hand side
+ * @throws IOException
+ */
+ public LogicalExpression getRhs() throws IOException {
+ return (LogicalExpression)plan.getSuccessors(this).get(2);
+ }
+
+ /**
+ * @link org.apache.pig.experimental.plan.Operator#accept(org.apache.pig.experimental.plan.PlanVisitor)
+ */
+ @Override
+ public void accept(PlanVisitor v) throws IOException {
+ if (!(v instanceof LogicalExpressionVisitor)) {
+ throw new IOException("Expected LogicalExpressionVisitor");
+ }
+ ((LogicalExpressionVisitor)v).visitBinCond(this);
+ }
+
+ @Override
+ public boolean isEqual(Operator other) {
+ if (other != null && other instanceof BinCondExpression) {
+ BinCondExpression ao = (BinCondExpression)other;
+ try {
+ return ao.getCondition().isEqual(getCondition()) &&
+ ao.getLhs().isEqual(getLhs()) && ao.getRhs().isEqual(getRhs());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ return false;
+ }
+ }
+
+}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ExpToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ExpToPhyTranslationVisitor.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ExpToPhyTranslationVisitor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ExpToPhyTranslationVisitor.java Mon Mar 15 03:28:27 2010
@@ -18,10 +18,13 @@
package org.apache.pig.experimental.logical.expression;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Stack;
+import org.apache.pig.ComparisonFunc;
+import org.apache.pig.EvalFunc;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.LogicalToPhysicalTranslatorException;
@@ -39,6 +42,7 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.Multiply;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.NotEqualToExpr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POAnd;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POBinCond;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POCast;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POIsNull;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POMapLookUp;
@@ -46,7 +50,10 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PONot;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POOr;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORegexp;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.PORelationToExprProject;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserComparisonFunc;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.Subtract;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.data.DataType;
@@ -56,6 +63,7 @@ import org.apache.pig.experimental.plan.
import org.apache.pig.experimental.plan.OperatorPlan;
import org.apache.pig.experimental.plan.PlanWalker;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.logicalLayer.LogicalOperator;
import org.apache.pig.impl.plan.NodeIdGenerator;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.impl.plan.PlanException;
@@ -234,48 +242,23 @@ public class ExpToPhyTranslationVisitor
@Override
public void visitProject(ProjectExpression op) throws IOException {
-// System.err.println("Entering Project");
POProject exprOp;
- if(op.getType() == DataType.BAG) {
- exprOp = new PORelationToExprProject(new OperatorKey(DEFAULT_SCOPE, nodeGen
- .getNextNodeId(DEFAULT_SCOPE)));
- } else {
+// if(op.getType() == DataType.BAG) {
+// exprOp = new PORelationToExprProject(new OperatorKey(DEFAULT_SCOPE, nodeGen
+// .getNextNodeId(DEFAULT_SCOPE)));
+// } else {
exprOp = new POProject(new OperatorKey(DEFAULT_SCOPE, nodeGen
.getNextNodeId(DEFAULT_SCOPE)));
- }
+// }
- // We dont have aliases in ExpressionOperators
- // exprOp.setAlias(op.getAlias());
exprOp.setResultType(op.getType());
exprOp.setColumn(op.getColNum());
- exprOp.setStar(false);
+ exprOp.setStar(op.isProjectStar());
// TODO implement this
// exprOp.setOverloaded(op.getOverloaded());
logToPhyMap.put(op, exprOp);
- currentPlan.add(exprOp);
-
- // We only have one input so connection is required from only one predecessor
-// PhysicalOperator from = logToPhyMap.get(op.findReferent(currentOp));
-// currentPlan.connect(from, exprOp);
-
-// List<Operator> predecessors = lp.getPredecessors(op);
-//
-// // Project might not have any predecessors
-// if (predecessors == null)
-// return;
-//
-// for (Operator lo : predecessors) {
-// PhysicalOperator from = logToPhyMap.get(lo);
-// try {
-// currentPlan.connect(from, exprOp);
-// } catch (PlanException e) {
-// int errCode = 2015;
-// String msg = "Invalid physical operators in the physical plan" ;
-// throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
-// }
-// }
-// System.err.println("Exiting Project");
+ currentPlan.add(exprOp);
}
@Override
@@ -351,7 +334,9 @@ public class ExpToPhyTranslationVisitor
logToPhyMap.put(op, pNot);
ExpressionOperator from = (ExpressionOperator) logToPhyMap.get(op
.getExpression());
- pNot.setResultType(op.getType());
+ pNot.setExpr(from);
+ pNot.setResultType(op.getType());
+ pNot.setOperandType(op.getType());
try {
currentPlan.connect(from, pNot);
} catch (PlanException e) {
@@ -371,7 +356,9 @@ public class ExpToPhyTranslationVisitor
logToPhyMap.put(op, pIsNull);
ExpressionOperator from = (ExpressionOperator) logToPhyMap.get(op
.getExpression());
- pIsNull.setResultType(op.getType());
+ pIsNull.setExpr(from);
+ pIsNull.setResultType(op.getType());
+ pIsNull.setOperandType(op.getType());
try {
currentPlan.connect(from, pIsNull);
} catch (PlanException e) {
@@ -409,6 +396,13 @@ public class ExpToPhyTranslationVisitor
}
@Override
+ public void visitRegex( RegexExpression op ) throws IOException {
+ BinaryExpressionOperator exprOp = new PORegexp(new OperatorKey(DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));
+
+ attachBinaryExpressionOperator(op, exprOp);
+ }
+
+ @Override
public void visitSubtract( SubtractExpression op ) throws IOException {
BinaryExpressionOperator exprOp = new Subtract(new OperatorKey(DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));
@@ -435,4 +429,85 @@ public class ExpToPhyTranslationVisitor
attachBinaryExpressionOperator(op, exprOp);
}
+
+ @Override
+ public void visitBinCond( BinCondExpression op ) throws IOException {
+
+ POBinCond exprOp = new POBinCond( new OperatorKey(DEFAULT_SCOPE,
+ nodeGen.getNextNodeId(DEFAULT_SCOPE)) );
+
+ exprOp.setResultType(op.getType());
+ exprOp.setCond((ExpressionOperator) logToPhyMap.get(op.getCondition()));
+ exprOp.setLhs((ExpressionOperator) logToPhyMap.get(op.getLhs()));
+ exprOp.setRhs((ExpressionOperator) logToPhyMap.get(op.getRhs()));
+ OperatorPlan oPlan = op.getPlan();
+
+ currentPlan.add(exprOp);
+ logToPhyMap.put(op, exprOp);
+
+ List<Operator> successors = oPlan.getSuccessors(op);
+ if (successors == null) {
+ return;
+ }
+ for (Operator lo : successors) {
+ PhysicalOperator from = logToPhyMap.get(lo);
+ try {
+ currentPlan.connect(from, exprOp);
+ } catch (PlanException e) {
+ int errCode = 2015;
+ String msg = "Invalid physical operators in the physical plan" ;
+ throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void visitUserFunc( UserFuncExpression op ) throws IOException {
+ Object f = PigContext.instantiateFuncFromSpec(op.getFuncSpec());
+ PhysicalOperator p;
+ if (f instanceof EvalFunc) {
+ p = new POUserFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
+ .getNextNodeId(DEFAULT_SCOPE)), -1,
+ null, op.getFuncSpec(), (EvalFunc) f);
+ } else {
+ p = new POUserComparisonFunc(new OperatorKey(DEFAULT_SCOPE, nodeGen
+ .getNextNodeId(DEFAULT_SCOPE)), -1,
+ null, op.getFuncSpec(), (ComparisonFunc) f);
+ }
+ p.setResultType(op.getType());
+ currentPlan.add(p);
+ List<LogicalExpression> fromList = op.getArguments();
+ if(fromList!=null){
+ for (LogicalExpression inputOperator : fromList) {
+ PhysicalOperator from = logToPhyMap.get(inputOperator);
+ try {
+ currentPlan.connect(from, p);
+ } catch (PlanException e) {
+ int errCode = 2015;
+ String msg = "Invalid physical operators in the physical plan" ;
+ throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
+ }
+ }
+ }
+ logToPhyMap.put(op, p);
+ }
+
+ @Override
+ public void visitBagDereference( BagDereferenceExpression op ) throws IOException {
+ POProject exprOp = new POProject(new OperatorKey(DEFAULT_SCOPE, nodeGen
+ .getNextNodeId(DEFAULT_SCOPE)));
+
+ exprOp.setResultType(op.getType());
+ exprOp.setColumns((ArrayList<Integer>)op.getBagColumns());
+ exprOp.setStar(false);
+ logToPhyMap.put(op, exprOp);
+ currentPlan.add(exprOp);
+
+ PhysicalOperator from = logToPhyMap.get( op.getProjectExpression() );
+
+ if( from != null ) {
+ currentPlan.connect(from, exprOp);
+ }
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/LogicalExpressionVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/LogicalExpressionVisitor.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/LogicalExpressionVisitor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/LogicalExpressionVisitor.java Mon Mar 15 03:28:27 2010
@@ -98,4 +98,17 @@ public abstract class LogicalExpressionV
public void visitMapLookup(MapLookupExpression op) throws IOException {
}
+
+ public void visitBinCond(BinCondExpression op) throws IOException {
+ }
+
+ public void visitUserFunc(UserFuncExpression op) throws IOException {
+ }
+
+ public void visitBagDereference(
+ BagDereferenceExpression bagDerefenceExpression) throws IOException {
+ }
+
+ public void visitRegex(RegexExpression op) throws IOException {
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ProjectExpression.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ProjectExpression.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ProjectExpression.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/ProjectExpression.java Mon Mar 15 03:28:27 2010
@@ -83,6 +83,10 @@ public class ProjectExpression extends C
}
+ public void setInputNum(int inputNum) {
+ input = inputNum;
+ }
+
/**
* Column number this project references. The column number is the column
* in the relational operator that contains this expression. The count
@@ -110,15 +114,20 @@ public class ProjectExpression extends C
this.type = type;
}
+ public boolean isProjectStar() {
+ return col<0;
+ }
+
@Override
public void setUid(LogicalRelationalOperator currentOp) throws IOException {
LogicalRelationalOperator referent = findReferent(currentOp);
LogicalSchema schema = referent.getSchema();
- if (schema != null) {
+ if (schema != null && !isProjectStar()) {
uid = schema.getField(col).uid;
} else {
- // If the schema of referent is null, we kindof create a uid so we
+ // If the schema of referent is null, or this is to project the whole tuple,
+ // we kindof create a uid so we
// can track it in remaining plan
uid = getNextUid();
}
@@ -133,7 +142,7 @@ public class ProjectExpression extends C
public LogicalRelationalOperator findReferent(LogicalRelationalOperator currentOp) throws IOException {
List<Operator> preds;
preds = currentOp.getPlan().getPredecessors(currentOp);
- if (preds == null || preds.size() - 1 < input) {
+ if (preds == null || input >= preds.size()) {
throw new IOException("Projection with nothing to reference!");
}
Added: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/RegexExpression.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/RegexExpression.java?rev=923043&view=auto
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/RegexExpression.java (added)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/RegexExpression.java Mon Mar 15 03:28:27 2010
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.experimental.logical.expression;
+
+import java.io.IOException;
+
+import org.apache.pig.experimental.plan.Operator;
+import org.apache.pig.experimental.plan.OperatorPlan;
+import org.apache.pig.experimental.plan.PlanVisitor;
+
+/**
+ * Regex Operator
+ */
+public class RegexExpression extends BinaryExpression {
+
+ /**
+ * Will add this operator to the plan and connect it to the
+ * left and right hand side operators.
+ * @param plan plan this operator is part of
+ * @param b this is the type of this expression
+ * @param lhs expression on its left hand side
+ * @param rhs expression on its right hand side
+ */
+ public RegexExpression(OperatorPlan plan,
+ byte b,
+ LogicalExpression lhs,
+ LogicalExpression rhs) {
+ super("Regex", plan, b, lhs, rhs);
+ }
+
+ /**
+ * @link org.apache.pig.experimental.plan.Operator#accept(org.apache.pig.experimental.plan.PlanVisitor)
+ */
+ @Override
+ public void accept(PlanVisitor v) throws IOException {
+ if (!(v instanceof LogicalExpressionVisitor)) {
+ throw new IOException("Expected LogicalExpressionVisitor");
+ }
+ ((LogicalExpressionVisitor)v).visitRegex(this);
+ }
+
+ @Override
+ public boolean isEqual(Operator other) {
+ if (other != null && other instanceof RegexExpression) {
+ RegexExpression ao = (RegexExpression)other;
+ try {
+ return ao.getLhs().isEqual(getLhs()) && ao.getRhs().isEqual(getRhs());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ } else {
+ return false;
+ }
+ }
+}
\ No newline at end of file
Added: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/UserFuncExpression.java?rev=923043&view=auto
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/UserFuncExpression.java (added)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/expression/UserFuncExpression.java Mon Mar 15 03:28:27 2010
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.experimental.logical.expression;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
+import org.apache.pig.PigException;
+import org.apache.pig.data.DataType;
+import org.apache.pig.experimental.plan.Operator;
+import org.apache.pig.experimental.plan.OperatorPlan;
+import org.apache.pig.experimental.plan.PlanVisitor;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.logicalLayer.ExpressionOperator;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.LOUserFunc;
+import org.apache.pig.impl.logicalLayer.LOVisitor;
+import org.apache.pig.impl.logicalLayer.LogicalOperator;
+import org.apache.pig.impl.logicalLayer.LogicalPlanCloner;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.VisitorException;
+
+public class UserFuncExpression extends LogicalExpression {
+
+ private FuncSpec mFuncSpec;
+
+ public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec, byte b) {
+ super("UserFunc", plan, b);
+ mFuncSpec = funcSpec;
+ plan.add(this);
+ }
+
+ public FuncSpec getFuncSpec() {
+ return mFuncSpec;
+ }
+
+ @Override
+ public void accept(PlanVisitor v) throws IOException {
+ if (!(v instanceof LogicalExpressionVisitor)) {
+ throw new IOException("Expected LogicalExpressionVisitor");
+ }
+ ((LogicalExpressionVisitor)v).visitUserFunc(this);
+ }
+
+ @Override
+ public boolean isEqual(Operator other) {
+ if( other instanceof UserFuncExpression ) {
+ UserFuncExpression exp = (UserFuncExpression)other;
+ return plan.isEqual(exp.plan) && mFuncSpec.equals(exp.mFuncSpec );
+ } else {
+ return false;
+ }
+ }
+
+ public List<LogicalExpression> getArguments() {
+ List<Operator> successors = null;
+ List<LogicalExpression> args = new ArrayList<LogicalExpression>();
+ try {
+ successors = plan.getSuccessors(this);
+
+ if(successors == null)
+ return args;
+
+ for(Operator lo : successors){
+ args.add((LogicalExpression)lo);
+ }
+ } catch (IOException e) {
+ return args;
+ }
+ return args;
+ }
+
+ /**
+ * @param funcSpec the FuncSpec to set
+ */
+ public void setFuncSpec(FuncSpec funcSpec) {
+ mFuncSpec = funcSpec;
+ }
+
+// @Override
+// public Schema.FieldSchema getFieldSchema() throws FrontendException {
+// if(!mIsFieldSchemaComputed) {
+// Schema inputSchema = new Schema();
+// List<ExpressionOperator> args = getArguments();
+// for(ExpressionOperator op: args) {
+// if (!DataType.isUsableType(op.getType())) {
+// mFieldSchema = null;
+// mIsFieldSchemaComputed = false;
+// int errCode = 1014;
+// String msg = "Problem with input: " + op + " of User-defined function: " + this ;
+// throw new FrontendException(msg, errCode, PigException.INPUT, false, null) ;
+// }
+// inputSchema.add(op.getFieldSchema());
+// }
+//
+// EvalFunc<?> ef = (EvalFunc<?>) PigContext.instantiateFuncFromSpec(mFuncSpec);
+// Schema udfSchema = ef.outputSchema(inputSchema);
+// byte returnType = DataType.findType(ef.getReturnType());
+//
+// if (null != udfSchema) {
+// Schema.FieldSchema fs;
+//// try {
+// if(udfSchema.size() == 0) {
+// fs = new Schema.FieldSchema(null, null, returnType);
+// } else if(udfSchema.size() == 1) {
+// fs = new Schema.FieldSchema(udfSchema.getField(0));
+// } else {
+// fs = new Schema.FieldSchema(null, udfSchema, DataType.TUPLE);
+// }
+//// } catch (ParseException pe) {
+//// throw new FrontendException(pe.getMessage());
+//// }
+// setType(fs.type);
+// mFieldSchema = fs;
+// mIsFieldSchemaComputed = true;
+// } else {
+// setType(returnType);
+// mFieldSchema = new Schema.FieldSchema(null, null, returnType);
+// mIsFieldSchemaComputed = true;
+// }
+// }
+// return mFieldSchema;
+// }
+}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllExpressionVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllExpressionVisitor.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllExpressionVisitor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllExpressionVisitor.java Mon Mar 15 03:28:27 2010
@@ -20,18 +20,22 @@ package org.apache.pig.experimental.logi
import java.io.IOException;
import java.util.Collection;
+import java.util.List;
import org.apache.pig.experimental.logical.expression.LogicalExpressionPlan;
import org.apache.pig.experimental.logical.expression.LogicalExpressionVisitor;
+import org.apache.pig.experimental.logical.relational.LOCogroup;
import org.apache.pig.experimental.logical.relational.LOFilter;
import org.apache.pig.experimental.logical.relational.LOForEach;
import org.apache.pig.experimental.logical.relational.LOGenerate;
import org.apache.pig.experimental.logical.relational.LOInnerLoad;
import org.apache.pig.experimental.logical.relational.LOJoin;
+import org.apache.pig.experimental.logical.relational.LOSplitOutput;
import org.apache.pig.experimental.logical.relational.LogicalPlanVisitor;
import org.apache.pig.experimental.logical.relational.LogicalRelationalOperator;
import org.apache.pig.experimental.plan.OperatorPlan;
import org.apache.pig.experimental.plan.PlanWalker;
+import org.apache.pig.impl.util.MultiMap;
/**
* A visitor that walks a logical plan and then applies a given
@@ -78,6 +82,19 @@ public abstract class AllExpressionVisit
}
@Override
+ public void visitLOCogroup(LOCogroup cg) throws IOException {
+ currentOp = cg;
+ MultiMap<Integer, LogicalExpressionPlan> expressionPlans = cg.getExpressionPlans();
+ for( Integer key : expressionPlans.keySet() ) {
+ Collection<LogicalExpressionPlan> exprPlans = expressionPlans.get(key);
+ for( LogicalExpressionPlan plan : exprPlans ) {
+ LogicalExpressionVisitor v = getVisitor(plan);
+ v.visit();
+ }
+ }
+ }
+
+ @Override
public void visitLOForEach(LOForEach foreach) throws IOException {
currentOp = foreach;
// We have an Inner OperatorPlan in ForEach, so we go ahead
@@ -99,13 +116,21 @@ public abstract class AllExpressionVisit
}
}
+ @Override
public void visitLOInnerLoad(LOInnerLoad load) throws IOException {
// the expression in LOInnerLoad contains info relative from LOForEach
// so use LOForeach as currentOp
currentOp = load.getLOForEach();
- LogicalExpressionPlan exp = load.getExpression();
+ LogicalExpressionPlan exp = (LogicalExpressionPlan)load.getProjection().getPlan();
LogicalExpressionVisitor v = getVisitor(exp);
v.visit();
}
+
+ @Override
+ public void visitLOSplitOutput(LOSplitOutput splitOutput) throws IOException {
+ currentOp = splitOutput;
+ LogicalExpressionVisitor v = getVisitor(splitOutput.getFilterPlan());
+ v.visit();
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllSameVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllSameVisitor.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllSameVisitor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/AllSameVisitor.java Mon Mar 15 03:28:27 2010
@@ -20,11 +20,15 @@ package org.apache.pig.experimental.logi
import java.io.IOException;
+import org.apache.pig.experimental.logical.relational.LOCogroup;
import org.apache.pig.experimental.logical.relational.LOFilter;
import org.apache.pig.experimental.logical.relational.LOForEach;
import org.apache.pig.experimental.logical.relational.LOJoin;
import org.apache.pig.experimental.logical.relational.LOLoad;
+import org.apache.pig.experimental.logical.relational.LOSplit;
+import org.apache.pig.experimental.logical.relational.LOSplitOutput;
import org.apache.pig.experimental.logical.relational.LOStore;
+import org.apache.pig.experimental.logical.relational.LOUnion;
import org.apache.pig.experimental.logical.relational.LogicalPlanVisitor;
import org.apache.pig.experimental.logical.relational.LogicalRelationalOperator;
import org.apache.pig.experimental.plan.OperatorPlan;
@@ -63,6 +67,11 @@ public abstract class AllSameVisitor ext
}
@Override
+ public void visitLOCogroup(LOCogroup cg) throws IOException {
+ execute(cg);
+ }
+
+ @Override
public void visitLOLoad(LOLoad load) throws IOException {
execute(load);
}
@@ -72,7 +81,23 @@ public abstract class AllSameVisitor ext
execute(store);
}
+ @Override
public void visitLOForEach(LOForEach foreach) throws IOException {
execute(foreach);
}
+
+ @Override
+ public void visitLOSplit(LOSplit split) throws IOException {
+ execute(split);
+ }
+
+ @Override
+ public void visitLOSplitOutput(LOSplitOutput splitOutput) throws IOException {
+ execute(splitOutput);
+ }
+
+ @Override
+ public void visitLOUnion(LOUnion union) throws IOException {
+ execute(union);
+ }
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/LogicalPlanOptimizer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/LogicalPlanOptimizer.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/LogicalPlanOptimizer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/LogicalPlanOptimizer.java Mon Mar 15 03:28:27 2010
@@ -22,6 +22,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.pig.experimental.logical.rules.AddForEach;
+import org.apache.pig.experimental.logical.rules.ColumnMapKeyPrune;
import org.apache.pig.experimental.logical.rules.FilterAboveForeach;
import org.apache.pig.experimental.logical.rules.MergeFilter;
import org.apache.pig.experimental.logical.rules.PushUpFilter;
@@ -77,20 +79,20 @@ public class LogicalPlanOptimizer extend
// Prune Set Marker
// This set is used for pruning columns and maps
- // This set only marks the items required for prunning
- // Next set does the prunning
-// s = new HashSet<Rule>();
-// ls.add(s);
-// // Add the PruneMap Filter
-// r = new PruneMapKeys("PruneMapKeys");
-// s.add(r);
-//
-// s = new HashSet<Rule>();
-// ls.add(s);
-// // Add the Prunner
-// r = new PruneMapKeys("PruneMapKeys");
-// s.add(r);
-
+
+ s = new HashSet<Rule>();
+ ls.add(s);
+ // Add the PruneMap Filter
+ r = new ColumnMapKeyPrune("ColumnMapKeyPrune");
+ s.add(r);
+
+ // Add LOForEach operator to trim off columns
+
+ s = new HashSet<Rule>();
+ ls.add(s);
+ // Add the AddForEach
+ r = new AddForEach("AddForEach");
+ s.add(r);
return ls;
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/PlanPrinter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/PlanPrinter.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/PlanPrinter.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/PlanPrinter.java Mon Mar 15 03:28:27 2010
@@ -28,6 +28,8 @@ import java.util.Stack;
import org.apache.pig.experimental.logical.expression.AddExpression;
import org.apache.pig.experimental.logical.expression.AndExpression;
+import org.apache.pig.experimental.logical.expression.BagDereferenceExpression;
+import org.apache.pig.experimental.logical.expression.BinCondExpression;
import org.apache.pig.experimental.logical.expression.CastExpression;
import org.apache.pig.experimental.logical.expression.ConstantExpression;
import org.apache.pig.experimental.logical.expression.DivideExpression;
@@ -48,13 +50,19 @@ import org.apache.pig.experimental.logic
import org.apache.pig.experimental.logical.expression.NotExpression;
import org.apache.pig.experimental.logical.expression.OrExpression;
import org.apache.pig.experimental.logical.expression.ProjectExpression;
+import org.apache.pig.experimental.logical.expression.RegexExpression;
import org.apache.pig.experimental.logical.expression.SubtractExpression;
+import org.apache.pig.experimental.logical.expression.UserFuncExpression;
+import org.apache.pig.experimental.logical.relational.LOCogroup;
import org.apache.pig.experimental.logical.relational.LOFilter;
import org.apache.pig.experimental.logical.relational.LOForEach;
import org.apache.pig.experimental.logical.relational.LOGenerate;
import org.apache.pig.experimental.logical.relational.LOInnerLoad;
import org.apache.pig.experimental.logical.relational.LOLoad;
+import org.apache.pig.experimental.logical.relational.LOSplit;
+import org.apache.pig.experimental.logical.relational.LOSplitOutput;
import org.apache.pig.experimental.logical.relational.LOStore;
+import org.apache.pig.experimental.logical.relational.LOUnion;
import org.apache.pig.experimental.logical.relational.LogicalRelationalOperator;
import org.apache.pig.experimental.plan.DepthFirstWalker;
import org.apache.pig.experimental.plan.Operator;
@@ -63,6 +71,7 @@ import org.apache.pig.experimental.plan.
import org.apache.pig.experimental.plan.PlanWalker;
import org.apache.pig.experimental.plan.ReverseDependencyOrderWalker;
import org.apache.pig.impl.plan.optimizer.Rule.WalkerAlgo;
+import org.apache.pig.impl.util.MultiMap;
public class PlanPrinter extends AllExpressionVisitor {
@@ -272,6 +281,26 @@ public class PlanPrinter extends AllExpr
public void visitDivide(DivideExpression exp) throws IOException {
simplevisit(exp);
}
+
+ @Override
+ public void visitBinCond(BinCondExpression exp ) throws IOException {
+ simplevisit(exp);
+ }
+
+ @Override
+ public void visitUserFunc(UserFuncExpression exp) throws IOException {
+ simplevisit(exp);
+ }
+
+ @Override
+ public void visitBagDereference(BagDereferenceExpression exp) throws IOException {
+ simplevisit(exp);
+ }
+
+ @Override
+ public void visitRegex(RegexExpression op) throws IOException {
+ simplevisit(op);
+ }
}
@Override
@@ -328,6 +357,47 @@ public class PlanPrinter extends AllExpr
printLevel();
stream.println( op.toString() );
}
+
+ @Override
+ public void visitLOCogroup(LOCogroup op) throws IOException {
+ printLevel();
+ stream.println( op.toString() );
+ MultiMap<Integer,LogicalExpressionPlan> exprPlans = op.getExpressionPlans();
+ for( Integer key : exprPlans.keySet() ) {
+ Collection<LogicalExpressionPlan> plans = exprPlans.get(key);
+ LogicalExpressionVisitor v = null;
+ level++;
+ for( LogicalExpressionPlan plan : plans ) {
+ v = getVisitor(plan);
+ v.visit();
+ }
+ level--;
+ }
+ }
+
+ @Override
+ public void visitLOSplitOutput(LOSplitOutput op) throws IOException {
+ printLevel();
+ stream.println( op.toString() );
+ LogicalExpressionVisitor v = getVisitor(op.getFilterPlan());
+ level++;
+ v.visit();
+ level--;
+ }
+
+ @Override
+ public void visitLOSplit(LOSplit op) throws IOException {
+ printLevel();
+ stream.println( op.toString() );
+ level++;
+ }
+
+ @Override
+ public void visitLOUnion(LOUnion op) throws IOException {
+ printLevel();
+ stream.println( op.toString() );
+ level++;
+ }
public String toString() {
return stream.toString();
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/ProjectionPatcher.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/ProjectionPatcher.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/ProjectionPatcher.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/ProjectionPatcher.java Mon Mar 15 03:28:27 2010
@@ -21,11 +21,13 @@ package org.apache.pig.experimental.logi
import java.io.IOException;
import java.util.List;
+import org.apache.pig.experimental.logical.expression.BagDereferenceExpression;
import org.apache.pig.experimental.logical.expression.LogicalExpressionPlan;
import org.apache.pig.experimental.logical.expression.LogicalExpressionVisitor;
import org.apache.pig.experimental.logical.expression.ProjectExpression;
import org.apache.pig.experimental.logical.relational.LogicalRelationalOperator;
import org.apache.pig.experimental.logical.relational.LogicalSchema;
+import org.apache.pig.experimental.plan.DependencyOrderWalker;
import org.apache.pig.experimental.plan.DepthFirstWalker;
import org.apache.pig.experimental.plan.Operator;
import org.apache.pig.experimental.plan.OperatorPlan;
@@ -58,6 +60,11 @@ public class ProjectionPatcher implement
@Override
public void visitProject(ProjectExpression p) throws IOException {
+ // if projection is for everything, just return
+ if (p.isProjectStar()) {
+ return;
+ }
+
// Get the uid for this projection. It must match the uid of the
// value it is projecting.
long myUid = p.getUid();
@@ -79,14 +86,13 @@ public class ProjectionPatcher implement
throw new IOException("Couldn't find matching uid for project");
}
p.setColNum(match);
- }
-
+ }
}
private static class ProjectionFinder extends AllExpressionVisitor {
public ProjectionFinder(OperatorPlan plan) {
- super(plan, new DepthFirstWalker(plan));
+ super(plan, new DependencyOrderWalker(plan));
}
@Override
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/SchemaPatcher.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/SchemaPatcher.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/SchemaPatcher.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/SchemaPatcher.java Mon Mar 15 03:28:27 2010
@@ -55,7 +55,10 @@ public class SchemaPatcher implements Pl
@Override
protected void execute(LogicalRelationalOperator op) throws IOException {
op.resetSchema();
- op.getSchema();
+
+ // can not call get schema at this point, because projections have not been
+ // adjusted. So only clean it up
+ // op.getSchema();
}
}
Modified: hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/UidStamper.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/UidStamper.java?rev=923043&r1=923042&r2=923043&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/UidStamper.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/experimental/logical/optimizer/UidStamper.java Mon Mar 15 03:28:27 2010
@@ -23,6 +23,8 @@ import java.util.List;
import org.apache.pig.experimental.logical.expression.AddExpression;
import org.apache.pig.experimental.logical.expression.AndExpression;
+import org.apache.pig.experimental.logical.expression.BagDereferenceExpression;
+import org.apache.pig.experimental.logical.expression.BinCondExpression;
import org.apache.pig.experimental.logical.expression.CastExpression;
import org.apache.pig.experimental.logical.expression.ConstantExpression;
import org.apache.pig.experimental.logical.expression.DivideExpression;
@@ -43,7 +45,9 @@ import org.apache.pig.experimental.logic
import org.apache.pig.experimental.logical.expression.NotExpression;
import org.apache.pig.experimental.logical.expression.OrExpression;
import org.apache.pig.experimental.logical.expression.ProjectExpression;
+import org.apache.pig.experimental.logical.expression.RegexExpression;
import org.apache.pig.experimental.logical.expression.SubtractExpression;
+import org.apache.pig.experimental.logical.expression.UserFuncExpression;
import org.apache.pig.experimental.logical.relational.LOLoad;
import org.apache.pig.experimental.logical.relational.LogicalSchema;
import org.apache.pig.experimental.logical.relational.LogicalSchema.LogicalFieldSchema;
@@ -154,17 +158,40 @@ public class UidStamper extends AllExpre
exp.setUid(currentOp);
}
+ @Override
public void visitMultiply(MultiplyExpression op) throws IOException {
op.setUid(currentOp);
}
+ @Override
public void visitMod(ModExpression op) throws IOException {
op.setUid(currentOp);
}
+ @Override
public void visitDivide(DivideExpression op) throws IOException {
op.setUid(currentOp);
}
+
+ @Override
+ public void visitBinCond(BinCondExpression op) throws IOException {
+ op.setUid(currentOp);
+ }
+
+ @Override
+ public void visitUserFunc(UserFuncExpression op) throws IOException {
+ op.setUid(currentOp);
+ }
+
+ @Override
+ public void visitBagDereference(BagDereferenceExpression op) throws IOException {
+ op.setUid(currentOp);
+ }
+
+ @Override
+ public void visitRegex(RegexExpression op) throws IOException {
+ op.setUid(currentOp);
+ }
}
/* (non-Javadoc)