You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/11/05 20:34:33 UTC
svn commit: r1198031 - in /pig/trunk: ./
src/org/apache/pig/newplan/logical/relational/
src/org/apache/pig/newplan/logical/rules/ test/org/apache/pig/test/
Author: daijy
Date: Sat Nov 5 19:34:32 2011
New Revision: 1198031
URL: http://svn.apache.org/viewvc?rev=1198031&view=rev
Log:
PIG-2346: TypeCastInsert should not insert Foreach if there is no as statement
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java
pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Nov 5 19:34:32 2011
@@ -290,6 +290,8 @@ BUG FIXES
PIG-2320: Error: "projection with nothing to reference" (daijy)
+PIG-2346: TypeCastInsert should not insert Foreach if there is no as statement (daijy)
+
Release 0.9.1 - Unreleased
IMPROVEMENTS
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOLoad.java Sat Nov 5 19:34:32 2011
@@ -278,4 +278,8 @@ public class LOLoad extends LogicalRelat
this.signature = signature;
loadFunc.setUDFContextSignature(signature);
}
+
+ public LogicalSchema getScriptSchema() {
+ return scriptSchema;
+ }
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOStream.java Sat Nov 5 19:34:32 2011
@@ -113,4 +113,8 @@ public class LOStream extends LogicalRel
public void resetUid() {
uidOnlySchema = null;
}
+
+ public LogicalSchema getScriptSchema() {
+ return scriptSchema;
+ }
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/TypeCastInserter.java Sat Nov 5 19:34:32 2011
@@ -77,9 +77,11 @@ public abstract class TypeCastInserter e
if (s == null) return false;
if (op instanceof LOLoad) {
+ if (((LOLoad)op).getScriptSchema()==null) return false;
if (((LOLoad)op).isCastInserted()) return false;
}
else {
+ if (((LOStream)op).getScriptSchema()==null) return false;
if (((LOStream)op).isCastInserted()) return false;
}
Modified: pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPartitionFilterPushDown.java Sat Nov 5 19:34:32 2011
@@ -568,7 +568,7 @@ public class TestPartitionFilterPushDown
Assert.assertTrue(!(iter.next() instanceof LOFilter));
counter++;
}
- Assert.assertEquals(counter, 7);
+ Assert.assertEquals(counter, 5);
}
//// helper methods ///////
Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1198031&r1=1198030&r2=1198031&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Sat Nov 5 19:34:32 2011
@@ -17,11 +17,18 @@
*/
package org.apache.pig.test;
+import java.io.IOException;
import java.util.Properties;
import junit.framework.Assert;
+import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.ExecType;
+import org.apache.pig.Expression;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.LoadMetadata;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.ResourceStatistics;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
@@ -29,17 +36,25 @@ import org.apache.pig.backend.hadoop.exe
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.expression.LogicalExpression;
import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOLoad;
import org.apache.pig.newplan.logical.relational.LOSort;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
+import org.apache.pig.parser.ParserException;
+import org.apache.pig.test.TestPartitionFilterPushDown.TestLoader;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -191,4 +206,49 @@ public class TestPlanGeneration extends
LogicalPlan lp = Util.parse(query, pc);
Util.optimizeNewLP(lp);
}
+
+ public static class SchemaLoader extends PigStorage implements LoadMetadata {
+
+ Schema schema;
+ public SchemaLoader(String schemaString) throws ParserException {
+ schema = Utils.getSchemaFromString(schemaString);
+ }
+ @Override
+ public ResourceSchema getSchema(String location, Job job)
+ throws IOException {
+ return new ResourceSchema(schema);
+ }
+
+ @Override
+ public ResourceStatistics getStatistics(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public String[] getPartitionKeys(String location, Job job)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public void setPartitionFilter(Expression partitionFilter)
+ throws IOException {
+ }
+ }
+
+ @Test
+ public void testLoaderWithSchema() throws Exception {
+ String query = "a = load 'foo' using " + SchemaLoader.class.getName() + "('name,age,gpa');\n"
+ + "b = filter a by age==20;"
+ + "store b into 'output';";
+ LogicalPlan lp = Util.parse(query, pc);
+ Util.optimizeNewLP(lp);
+
+ LOLoad loLoad = (LOLoad)lp.getSources().get(0);
+ LOFilter loFilter = (LOFilter)lp.getSuccessors(loLoad).get(0);
+ LOStore loStore = (LOStore)lp.getSuccessors(loFilter).get(0);
+
+ Assert.assertTrue(lp.getSuccessors(loStore)==null);
+ }
}