You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by th...@apache.org on 2010/09/04 00:28:40 UTC
svn commit: r992494 - in /hadoop/pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/impl/logicalLayer/parser/ src/org/apache/pig/newplan/
src/org/apache/pig/newplan/logical/expression/
src/org/apache/pig/newplan/logical/optimizer/
Author: thejas
Date: Fri Sep 3 22:28:39 2010
New Revision: 992494
URL: http://svn.apache.org/viewvc?rev=992494&view=rev
Log:
PIG-1595: casting relation to scalar- problem with handling of data from non PigStorage loaders
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/PigServer.java
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/trunk/src/org/apache/pig/newplan/DependencyOrderWalker.java
hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaPatcher.java
hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Sep 3 22:28:39 2010
@@ -195,6 +195,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1595: casting relation to scalar- problem with handling of data from non PigStorage loaders (thejas)
+
PIG-1591: pig does not create a log file, if tje MR job succeeds but front end fails (nrai via daijy)
PIG-1543: IsEmpty returns the wrong value after using LIMIT (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Fri Sep 3 22:28:39 2010
@@ -1261,9 +1261,13 @@ public class PigServer {
// If referredPlan already has a store,
// we just use it instead of adding one from our pocket
store = referredPlan.getLeaves().get(0);
- if(store instanceof LOStore) {
- // use this store
- fileSpec = ((LOStore)store).getOutputFile();
+ if(store instanceof LOStore
+ &&
+ ((LOStore)store).getOutputFile().getFuncName().equals(
+ InterStorage.class.getName())
+ ) {
+ // use this store
+ fileSpec = ((LOStore)store).getOutputFile();
}
else {
// add new store
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Fri Sep 3 22:28:39 2010
@@ -783,12 +783,12 @@ public class QueryParser {
lp.add(rconst);
lp.connect(rconst, expr);
- if(over != null && over.getField(colNum).type != DataType.BYTEARRAY) {
- LOCast loCast = new LOCast(lp, new OperatorKey(scope, getNextId()), over.getField(colNum).type);
- lp.add(loCast);
- lp.connect(expr, loCast);
- expr = loCast;
- }
+// if(over != null && over.getField(colNum).type != DataType.BYTEARRAY) {
+// LOCast loCast = new LOCast(lp, new OperatorKey(scope, getNextId()), over.getField(colNum).type);
+// lp.add(loCast);
+// lp.connect(expr, loCast);
+// expr = loCast;
+// }
return expr;
}
}
Modified: hadoop/pig/trunk/src/org/apache/pig/newplan/DependencyOrderWalker.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/newplan/DependencyOrderWalker.java?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/newplan/DependencyOrderWalker.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/newplan/DependencyOrderWalker.java Fri Sep 3 22:28:39 2010
@@ -64,7 +64,7 @@ public class DependencyOrderWalker exten
List<Operator> fifo = new ArrayList<Operator>();
Set<Operator> seen = new HashSet<Operator>();
- List<Operator> leaves = plan.getSinks();
+ List<Operator> leaves = getSinks();
if (leaves == null) return;
for (Operator op : leaves) {
doAllPredecessors(op, seen, fifo);
@@ -74,6 +74,10 @@ public class DependencyOrderWalker exten
op.accept(visitor);
}
}
+
+ protected List<Operator> getSinks() throws FrontendException{
+ return plan.getSinks();
+ }
protected void doAllPredecessors(Operator node,
Set<Operator> seen,
Modified: hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java Fri Sep 3 22:28:39 2010
@@ -511,7 +511,7 @@ public class ExpToPhyTranslationVisitor
//We need to track all the scalars
if(op.getImplicitReferencedOperator() != null) {
- scalarAliasMap.put(p, (LogicalRelationalOperator)op.getImplicitReferencedOperator());
+ scalarAliasMap.put(p, op.getImplicitReferencedOperator());
}
}
Modified: hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/newplan/logical/expression/UserFuncExpression.java Fri Sep 3 22:28:39 2010
@@ -41,7 +41,7 @@ import org.apache.pig.newplan.logical.re
public class UserFuncExpression extends LogicalExpression {
private FuncSpec mFuncSpec;
- private Operator implicitReferencedOperator = null;
+ private LogicalRelationalOperator implicitReferencedOperator = null;
public UserFuncExpression(OperatorPlan plan, FuncSpec funcSpec) {
super("UserFunc", plan);
@@ -108,12 +108,9 @@ public class UserFuncExpression extends
List<Operator> args = plan.getSuccessors(this);
if(args != null && args.size() > 0 ){
int pos = (Integer)((ConstantExpression)args.get(0)).getValue();
- LogicalRelationalOperator inp = (LogicalRelationalOperator)implicitReferencedOperator;
-
- if( inp.getSchema() != null){
- LogicalFieldSchema inpFs = inp.getSchema().getField(pos);
+ if( implicitReferencedOperator.getSchema() != null){
+ LogicalFieldSchema inpFs = implicitReferencedOperator.getSchema().getField(pos);
fieldSchema = new LogicalFieldSchema(inpFs);
- // fieldSchema.alias = "ReadScalars_" + fieldSchema.alias;
}else{
fieldSchema = new LogicalFieldSchema(null, null, DataType.BYTEARRAY);
}
@@ -157,11 +154,11 @@ public class UserFuncExpression extends
return fieldSchema;
}
- public Operator getImplicitReferencedOperator() {
+ public LogicalRelationalOperator getImplicitReferencedOperator() {
return implicitReferencedOperator;
}
- public void setImplicitReferencedOperator(Operator implicitReferencedOperator) {
+ public void setImplicitReferencedOperator(LogicalRelationalOperator implicitReferencedOperator) {
this.implicitReferencedOperator = implicitReferencedOperator;
}
Modified: hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaPatcher.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaPatcher.java?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaPatcher.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaPatcher.java Fri Sep 3 22:28:39 2010
@@ -20,6 +20,7 @@ package org.apache.pig.newplan.logical.o
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.optimizer.PlanTransformListener;
/**
Modified: hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java?rev=992494&r1=992493&r2=992494&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/newplan/logical/optimizer/SchemaResetter.java Fri Sep 3 22:28:39 2010
@@ -5,7 +5,7 @@ import java.util.List;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.MultiMap;
-import org.apache.pig.newplan.DependencyOrderWalker;
+import org.apache.pig.newplan.DependencyOrderWalkerLPScalar;
import org.apache.pig.newplan.OperatorPlan;
import org.apache.pig.newplan.PlanWalker;
import org.apache.pig.newplan.ReverseDependencyOrderWalker;
@@ -33,7 +33,7 @@ import org.apache.pig.newplan.logical.re
public class SchemaResetter extends LogicalRelationalNodesVisitor {
public SchemaResetter(OperatorPlan plan) throws FrontendException {
- super(plan, new DependencyOrderWalker(plan));
+ super(plan, new DependencyOrderWalkerLPScalar(plan));
}
@Override