You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/04/22 19:31:54 UTC
svn commit: r1095968 - in /pig/trunk: ./
src/org/apache/pig/newplan/logical/expression/
src/org/apache/pig/newplan/logical/relational/
src/org/apache/pig/newplan/logical/visitor/ src/org/apache/pig/parser/
test/org/apache/pig/test/
Author: daijy
Date: Fri Apr 22 17:31:53 2011
New Revision: 1095968
URL: http://svn.apache.org/viewvc?rev=1095968&view=rev
Log:
PIG-1910: incorrect schema shown when project-star is used with other projections
Added:
pig/trunk/src/org/apache/pig/newplan/logical/visitor/UDFFinder.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/expression/DereferenceExpression.java
pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/expression/ProjectExpression.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ColumnAliasConversionVisitor.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/test/org/apache/pig/test/TestPigServer.java
pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Apr 22 17:31:53 2011
@@ -172,6 +172,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1910: incorrect schema shown when project-star is used with other projections (daijy)
+
PIG-2005: Discrepancy in the way dry run handles semicolon in macro definition (rding)
PIG-1281: Detect org.apache.pig.data.DataByteArray cannot be cast to
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/DereferenceExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/DereferenceExpression.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/DereferenceExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/DereferenceExpression.java Fri Apr 22 17:31:53 2011
@@ -145,13 +145,16 @@ public class DereferenceExpression exten
// Get the tuple inner schema
LogicalSchema origSchema = predFS.schema.getField(0).schema;;
// Slice the tuple inner schema
- if (origSchema!=null && origSchema.size()!=0) {
- if (!rawColumns.isEmpty()) {
- columns = translateAliasToPos(origSchema, rawColumns);
- }
- for (int column:columns) {
+ if (!rawColumns.isEmpty()) {
+ columns = translateAliasToPos(origSchema, rawColumns);
+ }
+ for (int column:columns) {
+ if (origSchema!=null && origSchema.size()!=0) {
innerSchema.addField(origSchema.getField(column));
}
+ else {
+ innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
+ }
}
}
LogicalSchema bagSchema = new LogicalSchema();
@@ -165,7 +168,13 @@ public class DereferenceExpression exten
if (!rawColumns.isEmpty()) {
columns = translateAliasToPos(predFS.schema, rawColumns);
}
- fieldSchema = predFS.schema.getField(columns.get(0));
+ if (predFS.schema!=null && predFS.schema.size()!=0) {
+ fieldSchema = predFS.schema.getField(columns.get(0));
+ }
+ else {
+ fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
+ uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
+ }
} else{
fieldSchema = new LogicalFieldSchema(null, null, DataType.BYTEARRAY);
uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
@@ -179,7 +188,7 @@ public class DereferenceExpression exten
List<Integer> columns = new ArrayList<Integer>();
for( Object rawColumn : rawColumns ) {
if( rawColumn instanceof Integer ) {
- if ((Integer)rawColumn>=schema.size() || (Integer)rawColumn<0) {
+ if (schema!=null && ((Integer)rawColumn>=schema.size() || (Integer)rawColumn<0)) {
throw new FrontendException("Index "+rawColumn + " out of range in schema:" + schema.toString(false), 1127);
}
columns.add( (Integer)rawColumn );
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/ExpToPhyTranslationVisitor.java Fri Apr 22 17:31:53 2011
@@ -257,7 +257,10 @@ public class ExpToPhyTranslationVisitor
.getNextNodeId(DEFAULT_SCOPE)));
}
- exprOp.setResultType(op.getType());
+ if (op.getFieldSchema()==null && op.isRangeOrStarProject())
+ exprOp.setResultType(DataType.TUPLE);
+ else
+ exprOp.setResultType(op.getType());
if(op.isProjectStar()){
exprOp.setStar(op.isProjectStar());
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/expression/ProjectExpression.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/expression/ProjectExpression.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/expression/ProjectExpression.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/expression/ProjectExpression.java Fri Apr 22 17:31:53 2011
@@ -292,39 +292,47 @@ public class ProjectExpression extends C
fieldSchema.uid = innerLoads.get(0).getProjection().getFieldSchema().uid;
}
else {
+ // InnerLoad and source is not bag
if(schema == null){
- byte type = DataType.BYTEARRAY;
- if(findReferent() instanceof LOInnerLoad &&
- ((LOInnerLoad)findReferent()).getProjection().isRangeOrStarProject()){
- //if its project all from LOInnerLoad, consider it to be a tuple
- type = DataType.TUPLE;
+ // if we get here, it is range or starProject, otherwise, innerLoad will convert schema to non-null
+ if (isRangeProject && endCol!=-1){
+ LogicalSchema innerSchema = new LogicalSchema();
+ for(int i = startCol; i <= endCol; i++){
+ //schema is null, so null alias
+ innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
+ }
+ fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
+ } else {
+ fieldSchema = null;
}
- fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, type);
}
else{
fieldSchema = schema.getField(0);
}
}
- uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
+ if (fieldSchema!=null)
+ uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
}
else {
if (schema == null) {
- byte type = DataType.BYTEARRAY;
- LogicalSchema innerSchema = null;
- if(isProjectStar()){
- type = DataType.TUPLE;
- }else if(isRangeProject){
- type = DataType.TUPLE;
- if(endCol != -1){
- innerSchema = new LogicalSchema();
+ if(isRangeOrStarProject()) {
+ if (isRangeProject && endCol!=-1){
+ LogicalSchema innerSchema = new LogicalSchema();
for(int i = startCol; i <= endCol; i++){
//schema is null, so null alias
innerSchema.addField(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
}
+ fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, DataType.TUPLE);
}
+ else {
+ fieldSchema = null;
+ }
+ } else {
+ fieldSchema = new LogicalSchema.LogicalFieldSchema(null, null, DataType.BYTEARRAY);
}
- fieldSchema = new LogicalSchema.LogicalFieldSchema(null, innerSchema, type);
- uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
+
+ if (fieldSchema!=null)
+ uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
}
else {
int index = -1;
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOCogroup.java Fri Apr 22 17:31:53 2011
@@ -103,7 +103,8 @@ public class LOCogroup extends LogicalRe
private LogicalFieldSchema getPlanSchema( LogicalExpressionPlan exprPlan ) throws FrontendException {
LogicalExpression sourceExp = (LogicalExpression) exprPlan.getSources().get(0);
LogicalFieldSchema planSchema = null;
- planSchema = sourceExp.getFieldSchema().deepCopy();
+ if (sourceExp.getFieldSchema()!=null)
+ planSchema = sourceExp.getFieldSchema().deepCopy();
return planSchema;
}
@@ -162,11 +163,11 @@ public class LOCogroup extends LogicalRe
Collection<LogicalExpressionPlan> plans = mExpressionPlans.get(key);
for( LogicalExpressionPlan plan : plans ) {
groupKeySchema = getPlanSchema(plan);
- // if any plan schema is null, that means we can't calculate
- // further schemas so we bail out
+ // if any plan schema is null, that means we cannot figure out
+ // the arity of keys, just give an empty tuple
if( groupKeySchema == null ) {
- schema = null;
- return schema;
+ groupKeySchema = new LogicalSchema.LogicalFieldSchema("group", null, DataType.TUPLE);
+ break;
}
groupKeySchema = new LogicalSchema.LogicalFieldSchema(groupKeySchema);
// Change the uid of this field
Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/ColumnAliasConversionVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ColumnAliasConversionVisitor.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/ColumnAliasConversionVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/ColumnAliasConversionVisitor.java Fri Apr 22 17:31:53 2011
@@ -80,7 +80,7 @@ public class ColumnAliasConversionVisito
for( Object rc : rawCols ) {
if( rc instanceof Integer ) {
col = (Integer)rc;
- if( schema != null && col >= schema.size() ) {
+ if( schema != null && schema.size()!=0 && col >= schema.size() ) {
throw new PlanValidationException( expr, "Out of bound access. Trying to access non-existent column: " +
col + ". Schema " + schema.toString(false) + " has " + schema.size() + " column(s).", 1000 );
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/LineageFindRelVisitor.java Fri Apr 22 17:31:53 2011
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.pig.FuncSpec;
import org.apache.pig.PigException;
import org.apache.pig.data.DataType;
+import org.apache.pig.impl.builtin.IdentityColumn;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.streaming.StreamingCommand;
@@ -45,6 +46,7 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.MapLookupExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.expression.ScalarExpression;
+import org.apache.pig.newplan.logical.expression.UserFuncExpression;
import org.apache.pig.newplan.logical.relational.LOCogroup;
import org.apache.pig.newplan.logical.relational.LOCross;
import org.apache.pig.newplan.logical.relational.LODistinct;
@@ -207,26 +209,38 @@ public class LineageFindRelVisitor exten
*/
private FuncSpec getAssociatedLoadFunc(LogicalRelationalOperator relOp) throws FrontendException {
LogicalSchema schema = relOp.getSchema();
+ FuncSpec funcSpec = null;
if(schema != null){
if(schema.size() == 0)
return null;
- FuncSpec funcSpec = uid2LoadFuncMap.get(schema.getField(0).uid);
- if(funcSpec == null)
- return null;
- for(int i=1; i<schema.size(); i++){
- LogicalFieldSchema fs = schema.getField(i);
- if(! funcSpec.equals(uid2LoadFuncMap.get(fs.uid))){
- //all uid are not associated with same func spec, there is no
- // single func spec that represents all the fields
- return null;
+ funcSpec = uid2LoadFuncMap.get(schema.getField(0).uid);
+ if(funcSpec != null) {
+ for(int i=1; i<schema.size(); i++){
+ LogicalFieldSchema fs = schema.getField(i);
+ if(! funcSpec.equals(uid2LoadFuncMap.get(fs.uid))){
+ //all uid are not associated with same func spec, there is no
+ // single func spec that represents all the fields
+ funcSpec = null;
+ break;
+ }
}
}
- return funcSpec;
}
- else{
- return rel2InputFuncMap.get(relOp);
+
+ if(funcSpec == null){
+ // If relOp is LOForEach and contains UDF, byte field could come from UDF.
+ // We don't assume it share the LoadCaster with predecessor
+ if (relOp instanceof LOForEach) {
+ UDFFinder udfFinder = new UDFFinder(((LOForEach) relOp).getInnerPlan());
+ udfFinder.visit();
+ if (udfFinder.getUDFList().size()!=0)
+ return null;
+ }
+
+ funcSpec = rel2InputFuncMap.get(relOp);
}
+ return funcSpec;
}
private void mapRelToPredLoadFunc(LogicalRelationalOperator relOp,
@@ -261,9 +275,16 @@ public class LineageFindRelVisitor exten
}
LogicalSchema sch = group.getSchema();
+
//if the group plans are associated with same load function , associate
//same load fucntion with group column schema
- mapMatchLoadFuncToUid(sch.getField(0), groupPlanSchemas);
+ if (getAssociatedLoadFunc(group)!=null) {
+ addUidLoadFuncToMap(sch.getField(0).uid, rel2InputFuncMap.get(group));
+ if (sch.getField(0).schema!=null)
+ setLoadFuncForUids(sch.getField(0).schema, rel2InputFuncMap.get(group));
+ }
+ else
+ mapMatchLoadFuncToUid(sch.getField(0), groupPlanSchemas);
@@ -467,6 +488,12 @@ public class LineageFindRelVisitor exten
return;
}
+ // If schema of any input is null, we skip output schema
+ for (LogicalFieldSchema fs : inputFieldSchemas) {
+ if (fs==null)
+ return;
+ }
+
//if same non null load func is associated with all fieldschemas
// asssociate that with the uid of outFS
LogicalFieldSchema inpFS1 = inputFieldSchemas.get(0);
@@ -531,6 +558,9 @@ public class LineageFindRelVisitor exten
// with the relation
LogicalRelationalOperator inputRel = proj.findReferent();
+ if (proj.getFieldSchema()==null)
+ return;
+
long uid = proj.getFieldSchema().uid;
if(uid2LoadFuncMap.get(uid) == null && (inputRel.getSchema() == null || inputRel instanceof LOInnerLoad)){
FuncSpec funcSpec = rel2InputFuncMap.get(inputRel);
@@ -538,7 +568,6 @@ public class LineageFindRelVisitor exten
addUidLoadFuncToMap(uid, funcSpec);
}
}
-
}
@Override
Added: pig/trunk/src/org/apache/pig/newplan/logical/visitor/UDFFinder.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/UDFFinder.java?rev=1095968&view=auto
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/UDFFinder.java (added)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/UDFFinder.java Fri Apr 22 17:31:53 2011
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.newplan.logical.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.newplan.DependencyOrderWalker;
+import org.apache.pig.newplan.OperatorPlan;
+import org.apache.pig.newplan.PlanWalker;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
+import org.apache.pig.newplan.logical.expression.LogicalExpressionVisitor;
+import org.apache.pig.newplan.logical.expression.UserFuncExpression;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOGenerate;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalNodesVisitor;
+
+public class UDFFinder extends LogicalRelationalNodesVisitor {
+
+ private List<UserFuncExpression> mUDFList = new ArrayList<UserFuncExpression>();
+
+ public UDFFinder(OperatorPlan plan)
+ throws FrontendException {
+ super(plan, new DependencyOrderWalker(plan));
+ }
+
+ @Override
+ public void visit(LOForEach foreach) throws FrontendException {
+ OperatorPlan innerPlan = foreach.getInnerPlan();
+ PlanWalker newWalker = currentWalker.spawnChildWalker(innerPlan);
+ pushWalker(newWalker);
+ currentWalker.walk(this);
+ popWalker();
+ }
+
+ @Override
+ public void visit(LOGenerate generate) throws FrontendException {
+ for (LogicalExpressionPlan plan : generate.getOutputPlans()) {
+ UDFExpFinder udfExpFinder = new UDFExpFinder(plan);
+ udfExpFinder.visit();
+ mUDFList.addAll(udfExpFinder.getUDFList());
+ }
+ }
+
+ /**
+ *
+ * @return true if the plan had any UDFs; false otherwise
+ */
+ public List<UserFuncExpression> getUDFList() {
+ return mUDFList;
+ }
+}
+
+class UDFExpFinder extends LogicalExpressionVisitor {
+
+ List<UserFuncExpression> mUDFList = new ArrayList<UserFuncExpression>();
+
+ UDFExpFinder(OperatorPlan plan)
+ throws FrontendException {
+ super(plan, new DependencyOrderWalker(plan));
+ }
+
+ @Override
+ public void visit(UserFuncExpression userFunc) {
+ mUDFList.add(userFunc);
+ }
+
+ public List<UserFuncExpression> getUDFList() {
+ return mUDFList;
+ }
+}
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Fri Apr 22 17:31:53 2011
@@ -99,7 +99,7 @@ public class QueryParserDriver {
} catch(RecognitionException ex) {
throw new ParserException( ex );
} catch(Exception ex) {
- throw new ParserException( ex.getMessage() );
+ throw new ParserException( ex.getMessage(), ex );
}
return plan;
Modified: pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServer.java Fri Apr 22 17:31:53 2011
@@ -510,7 +510,7 @@ public class TestPigServer {
InputStream fileWithStdOutContents = new DataInputStream( new BufferedInputStream( new FileInputStream(stdOutRedirectedFile)));
BufferedReader reader = new BufferedReader(new InputStreamReader(fileWithStdOutContents));
while ((s = reader.readLine()) != null) {
- Assert.assertTrue(s.equals("b: {(null)}"));
+ Assert.assertTrue(s.equals("Schema for b unknown."));
}
fileWithStdOutContents.close();
}
Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Fri Apr 22 17:31:53 2011
@@ -19,15 +19,27 @@ package org.apache.pig.test;
import java.util.Properties;
+import junit.framework.Assert;
+
import org.apache.pig.ExecType;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POSort;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.expression.LogicalExpression;
+import org.apache.pig.newplan.logical.relational.LOCogroup;
+import org.apache.pig.newplan.logical.relational.LOForEach;
+import org.apache.pig.newplan.logical.relational.LOSort;
import org.apache.pig.newplan.logical.relational.LOStore;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -43,6 +55,107 @@ public class TestPlanGeneration extends
}
@Test
+ public void testGenerateStar() throws Exception {
+ String query = "a = load 'x';" +
+ "b = foreach a generate *;" +
+ "store b into '111';";
+
+ LogicalPlan lp = Util.parseAndPreprocess(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
+ assert(loForEach.getSchema()==null);
+ }
+
+ @Test
+ public void testEmptyBagDereference() throws Exception {
+ String query = "A = load 'x' as ( u:bag{} );" +
+ "B = foreach A generate u.$100;" +
+ "store B into '111';";
+
+ LogicalPlan lp = Util.parseAndPreprocess(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
+ LogicalSchema schema = loForEach.getSchema();
+ Assert.assertTrue(schema.size()==1);
+ LogicalFieldSchema bagFieldSchema = schema.getField(0);
+ Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
+ LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
+ Assert.assertTrue(tupleFieldSchema.schema.size()==1);
+ Assert.assertTrue(tupleFieldSchema.schema.getField(0).type==DataType.BYTEARRAY);
+ }
+
+ @Test
+ public void testEmptyTupleDereference() throws Exception {
+ String query = "A = load 'x' as ( u:tuple() );" +
+ "B = foreach A generate u.$100;" +
+ "store B into '111';";
+
+ LogicalPlan lp = Util.parseAndPreprocess(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
+ LogicalSchema schema = loForEach.getSchema();
+ Assert.assertTrue(schema.size()==1);
+ Assert.assertTrue(schema.getField(0).type==DataType.BYTEARRAY);
+ }
+
+ @Test
+ public void testEmptyBagInnerPlan() throws Exception {
+ String query = "A = load 'x' as ( u:bag{} );" +
+ "B = foreach A { B1 = filter u by $1==0; generate B1;};" +
+ "store B into '111';";
+
+ LogicalPlan lp = Util.parseAndPreprocess(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ LOForEach loForEach = (LOForEach)lp.getPredecessors(loStore).get(0);
+ LogicalSchema schema = loForEach.getSchema();
+ Assert.assertTrue(schema.size()==1);
+ LogicalFieldSchema bagFieldSchema = schema.getField(0);
+ Assert.assertTrue(bagFieldSchema.type==DataType.BAG);
+ LogicalFieldSchema tupleFieldSchema = bagFieldSchema.schema.getField(0);
+ Assert.assertTrue(tupleFieldSchema.schema==null);
+ }
+
+ @Test
+ public void testOrderByNullFieldSchema() throws Exception {
+ String query = "A = load 'x';" +
+ "B = order A by *;" +
+ "store B into '111';";
+
+ LogicalPlan lp = Util.parseAndPreprocess(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ LOSort loSort = (LOSort)lp.getPredecessors(loStore).get(0);
+ Operator sortPlanLeaf = loSort.getSortColPlans().get(0).getSources().get(0);
+ LogicalFieldSchema sortPlanFS = ((LogicalExpression)sortPlanLeaf).getFieldSchema();
+ Assert.assertTrue(sortPlanFS==null);
+
+ PhysicalPlan pp = Util.buildPhysicalPlanFromNewLP(lp, pc);
+ POStore poStore = (POStore)pp.getLeaves().get(0);
+ POSort poSort = (POSort)pp.getPredecessors(poStore).get(0);
+ POProject poProject = (POProject)poSort.getSortPlans().get(0).getLeaves().get(0);
+ Assert.assertTrue(poProject.getResultType()==DataType.TUPLE);
+ }
+
+ @Test
+ public void testGroupByNullFieldSchema() throws Exception {
+ String query = "A = load 'x';" +
+ "B = group A by *;" +
+ "store B into '111';";
+
+ LogicalPlan lp = Util.parseAndPreprocess(query, pc);
+ Util.optimizeNewLP(lp);
+ LOStore loStore = (LOStore)lp.getSinks().get(0);
+ LOCogroup loCoGroup = (LOCogroup)lp.getPredecessors(loStore).get(0);
+ LogicalFieldSchema groupFieldSchema = loCoGroup.getSchema().getField(0);
+ Assert.assertTrue(groupFieldSchema.type==DataType.TUPLE);
+ Assert.assertTrue(groupFieldSchema.schema==null);
+ }
+
+ @Test
public void testStoreAlias() throws Exception {
String query = "A = load 'data' as (a0, a1);" +
"B = filter A by a0 > 1;" +
Modified: pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestTypeCheckingValidatorNewLP.java Fri Apr 22 17:31:53 2011
@@ -4069,5 +4069,21 @@ public class TestTypeCheckingValidatorNe
}
+ public static class TestUDFTupleNullInnerSchema extends EvalFunc<Tuple> {
+ @Override
+ public Tuple exec(Tuple input) throws IOException {
+ return null;
+ }
+ }
+
+ @Test
+ public void testUDFNoInnerSchema() throws FrontendException {
+ String query = "a= load '1.txt';"
+ + "b = foreach a generate "+TestUDFTupleNullInnerSchema.class.getName()+"($0);"
+ + "c = foreach b generate flatten($0);"
+ + "d = foreach c generate $0 + 1;";
+
+ checkLastForeachCastLoadFunc(query, null, 0);
+ }
}
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1095968&r1=1095967&r2=1095968&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Fri Apr 22 17:31:53 2011
@@ -947,4 +947,22 @@ public class Util {
new CastLineageSetter(lp, collector).visit();
return lp;
}
+
+ public static org.apache.pig.newplan.logical.relational.LogicalPlan parseAndPreprocess(String query, PigContext pc) throws FrontendException {
+ Map<String, String> fileNameMap = new HashMap<String, String>();
+ QueryParserDriver parserDriver = new QueryParserDriver( pc, "test", fileNameMap );
+ org.apache.pig.newplan.logical.relational.LogicalPlan lp = parserDriver.parse( query );
+
+ new ColumnAliasConversionVisitor( lp ).visit();
+ new SchemaAliasVisitor( lp ).visit();
+ new ScalarVisitor( lp, pc ).visit();
+
+ CompilationMessageCollector collector = new CompilationMessageCollector() ;
+
+ new TypeCheckingRelVisitor( lp, collector).visit();
+
+ new UnionOnSchemaSetter( lp ).visit();
+ new CastLineageSetter(lp, collector).visit();
+ return lp;
+ }
}