You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/01/12 03:30:37 UTC
svn commit: r1057974 - in /pig/trunk: ./
src/org/apache/pig/newplan/logical/rules/
src/org/apache/pig/newplan/optimizer/ test/org/apache/pig/test/
Author: daijy
Date: Wed Jan 12 02:30:37 2011
New Revision: 1057974
URL: http://svn.apache.org/viewvc?rev=1057974&view=rev
Log:
PIG-1785: New logical plan: uid conflict in flattened fields
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java
pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java
pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java
pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jan 12 02:30:37 2011
@@ -257,6 +257,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1785: New logical plan: uid conflict in flattened fields (daijy)
+
PIG-1787: Error in logical plan generated (daijy)
PIG-1791: System property mapred.output.compress, but pig-cluster-hadoop-site.xml doesn't (daijy)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/DuplicateForEachColumnRewrite.java Wed Jan 12 02:30:37 2011
@@ -25,6 +25,7 @@ import java.util.List;
import java.util.Map;
import org.apache.pig.FuncSpec;
+import org.apache.pig.data.DataType;
import org.apache.pig.impl.builtin.IdentityColumn;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.Pair;
@@ -35,6 +36,8 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.LogicalExpressionPlan;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
import org.apache.pig.newplan.logical.expression.UserFuncExpression;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
+import org.apache.pig.newplan.logical.optimizer.UidResetter;
import org.apache.pig.newplan.logical.relational.LOCross;
import org.apache.pig.newplan.logical.relational.LOForEach;
import org.apache.pig.newplan.logical.relational.LOGenerate;
@@ -44,6 +47,7 @@ import org.apache.pig.newplan.logical.re
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.relational.LogicalSchema;
+import org.apache.pig.newplan.logical.relational.LogicalSchema.LogicalFieldSchema;
import org.apache.pig.newplan.logical.rules.PushDownForEachFlatten.PushDownForEachFlattenTransformer;
import org.apache.pig.newplan.optimizer.Rule;
import org.apache.pig.newplan.optimizer.Transformer;
@@ -56,6 +60,8 @@ public class DuplicateForEachColumnRewri
public DuplicateForEachColumnRewrite(String n) {
super(n, true);
+ // See comments in ImplicitSplitInserter for the reason to skip listener
+ setSkipListener(true);
}
@Override
@@ -81,18 +87,48 @@ public class DuplicateForEachColumnRewri
LOGenerate gen = (LOGenerate)foreach.getInnerPlan().getSinks().get(0);
List<LogicalExpressionPlan> expPlans = gen.getOutputPlans();
+ boolean[] flattens = gen.getFlattenFlags();
List<Long> uidSeen = new ArrayList<Long>();
- for (LogicalExpressionPlan expPlan : expPlans) {
+ for (int i=0;i<expPlans.size();i++) {
+ LogicalExpressionPlan expPlan = expPlans.get(i);
+ boolean flatten = flattens[i];
LogicalExpression exp = (LogicalExpression)expPlan.getSources().get(0);
if (exp.getFieldSchema()!=null) {
- long uid = exp.getFieldSchema().uid;
- if (uidSeen.contains(uid)) {
- expPlansToInsertIdentity.add(expPlan);
+ if (flatten && (exp.getFieldSchema().type == DataType.BAG || exp.getFieldSchema().type == DataType.TUPLE)) {
+ List<LogicalFieldSchema> innerFieldSchemas = null;
+ if (exp.getFieldSchema().type == DataType.BAG) {
+ if (exp.getFieldSchema().schema!=null) {
+ if (exp.getFieldSchema().schema.isTwoLevelAccessRequired()) {
+ // assert(fieldSchema.schema.size() == 1 && fieldSchema.schema.getField(0).type == DataType.TUPLE)
+ innerFieldSchemas = exp.getFieldSchema().schema.getField(0).schema.getFields();
+ } else {
+ innerFieldSchemas = exp.getFieldSchema().schema.getFields();
+ }
+ }
+ }
+ else { // DataType.TUPLE
+ if (exp.getFieldSchema().schema!=null)
+ innerFieldSchemas = exp.getFieldSchema().schema.getFields();
+ }
+ if (innerFieldSchemas != null) {
+ for (LogicalFieldSchema innerFieldSchema : innerFieldSchemas) {
+ long uid = innerFieldSchema.uid;
+ if (checkAndAdd(uid, uidSeen)) {
+ // Seen before
+ expPlansToInsertIdentity.add(expPlan);
+ break;
+ }
+ }
+ }
}
else {
- uidSeen.add(uid);
+ long uid = exp.getFieldSchema().uid;
+ if (checkAndAdd(uid, uidSeen)) {
+ // Seen before
+ expPlansToInsertIdentity.add(expPlan);
+ }
}
}
}
@@ -101,7 +137,14 @@ public class DuplicateForEachColumnRewri
return false;
return true;
- }
+ }
+
+ private boolean checkAndAdd(long uid, List<Long> uidSeen) {
+ if (uidSeen.contains(uid))
+ return true;
+ uidSeen.add(uid);
+ return false;
+ }
@Override
public OperatorPlan reportChanges() {
@@ -116,6 +159,14 @@ public class DuplicateForEachColumnRewri
expPlan.connect(userFuncExpression, oldRoot);
}
expPlansToInsertIdentity.clear();
+
+ // Since we adjust the uid layout, clear all cached uids
+ UidResetter uidResetter = new UidResetter(currentPlan);
+ uidResetter.visit();
+
+ // Manually regenerate schema since we skip listener
+ SchemaResetter schemaResetter = new SchemaResetter(currentPlan);
+ schemaResetter.visit();
}
}
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/ImplicitSplitInserter.java Wed Jan 12 02:30:37 2011
@@ -26,6 +26,7 @@ import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.Pair;
import org.apache.pig.newplan.Operator;
+import org.apache.pig.newplan.logical.optimizer.SchemaResetter;
import org.apache.pig.newplan.logical.optimizer.UidResetter;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.OperatorPlan;
@@ -49,6 +50,11 @@ public class ImplicitSplitInserter exten
public ImplicitSplitInserter(String n) {
super(n, true);
+ // Skip listener, especially, skip ProjectionPatcher so that we can keep column reference for
+ // ProjectExpression (Once ProjectionPatcher is invoked, column reference will be gone in favor of uid reference).
+ // There is no need for ProjectionPatcher in this rule since we don't swap columns; however, uid conflict is not solved
+ // at this moment (until after DuplicateForEachColumnRewrite), so keep column reference for now
+ setSkipListener(true);
}
@Override
@@ -128,8 +134,14 @@ public class ImplicitSplitInserter exten
currentPlan.connect(splitOp, splitOutput);
currentPlan.connect(splitOutput, pos.first, suc, pos.second);
}
+
+ // Since we adjust the uid layout, clear all cached uids
UidResetter uidResetter = new UidResetter(currentPlan);
uidResetter.visit();
+
+ // Manually regenerate schema since we skip listener
+ SchemaResetter schemaResetter = new SchemaResetter(currentPlan);
+ schemaResetter.visit();
}
@Override
Modified: pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/optimizer/PlanOptimizer.java Wed Jan 12 02:30:37 2011
@@ -108,8 +108,10 @@ public abstract class PlanOptimizer {
if (transformer.check(m)) {
sawMatch = true;
transformer.transform(m);
- for(PlanTransformListener l: listeners) {
- l.transformed(plan, transformer.reportChanges());
+ if (!rule.isSkipListener()) {
+ for(PlanTransformListener l: listeners) {
+ l.transformed(plan, transformer.reportChanges());
+ }
}
}
} catch (Exception e) {
Modified: pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/optimizer/Rule.java Wed Jan 12 02:30:37 2011
@@ -51,6 +51,7 @@ public abstract class Rule {
protected static final Log log = LogFactory.getLog(Rule.class);
private transient Set<Operator> matchedNodes = new HashSet<Operator>();
private boolean mandatory;
+ private boolean skipListener = false;
/**
* Create this rule by using the default pattern that this rule provided
@@ -95,6 +96,14 @@ public abstract class Rule {
return pattern;
}
+ protected boolean isSkipListener() {
+ return this.skipListener;
+ }
+
+ protected void setSkipListener(boolean skip) {
+ this.skipListener = skip;
+ }
+
/**
* Search for all the sub-plans that matches the pattern
* defined by this rule.
Modified: pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1057974&r1=1057973&r2=1057974&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Wed Jan 12 02:30:37 2011
@@ -989,7 +989,7 @@ public class TestEvalPipeline2 extends T
// See PIG-1766
@Test
- public void testForEachSameOriginColumn() throws Exception{
+ public void testForEachSameOriginColumn1() throws Exception{
String[] input1 = {
"1\t2",
"1\t3",
@@ -1002,10 +1002,10 @@ public class TestEvalPipeline2 extends T
"2\ttwo",
};
- Util.createInputFile(cluster, "table_testForEachSameOriginColumn1", input1);
- Util.createInputFile(cluster, "table_testForEachSameOriginColumn2", input2);
- pigServer.registerQuery("A = load 'table_testForEachSameOriginColumn1' AS (a0:int, a1:int);");
- pigServer.registerQuery("B = load 'table_testForEachSameOriginColumn2' AS (b0:int, b1:chararray);");
+ Util.createInputFile(cluster, "table_testForEachSameOriginColumn1_1", input1);
+ Util.createInputFile(cluster, "table_testForEachSameOriginColumn1_2", input2);
+ pigServer.registerQuery("A = load 'table_testForEachSameOriginColumn1_1' AS (a0:int, a1:int);");
+ pigServer.registerQuery("B = load 'table_testForEachSameOriginColumn1_2' AS (b0:int, b1:chararray);");
pigServer.registerQuery("C = join A by a0, B by b0;");
pigServer.registerQuery("D = foreach B generate b0 as d0, b1 as d1;");
pigServer.registerQuery("E = join C by a1, D by d0;");
@@ -1156,4 +1156,46 @@ public class TestEvalPipeline2 extends T
assertTrue(t.toString().equals("(1,2,1,2)"));
assertFalse(iter.hasNext());
}
+
+ // See PIG-1785
+ @Test
+ public void testForEachSameOriginColumn2() throws Exception{
+ String[] input1 = {
+ "{(1,2),(2,3)}",
+ };
+
+ Util.createInputFile(cluster, "table_testForEachSameOriginColumn2", input1);
+
+ pigServer.registerQuery("a = load 'table_testForEachSameOriginColumn2' as (a0:bag{t:tuple(i0:int, i1:int)});");
+ pigServer.registerQuery("b = foreach a generate flatten(a0) as (b0, b1), flatten(a0) as (b2, b3);");
+ pigServer.registerQuery("c = filter b by b0>b2;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("c");
+
+ Tuple t = iter.next();
+ assertTrue(t.toString().contains("(2,3,1,2)"));
+ assertFalse(iter.hasNext());
+ }
+
+ // See PIG-1785
+ @Test
+ public void testForEachSameOriginColumn3() throws Exception{
+ String[] input1 = {
+ "1\t1\t2",
+ "1\t2\t3",
+ };
+
+ Util.createInputFile(cluster, "table_testForEachSameOriginColumn3", input1);
+
+ pigServer.registerQuery("a = load 'table_testForEachSameOriginColumn3' as (a0:int, a1:int, a2:int);");
+ pigServer.registerQuery("b = group a by a0;");
+ pigServer.registerQuery("c = foreach b generate flatten(a.(a1,a2)) as (b0, b1), flatten(a.(a1,a2)) as (b2, b3);");
+ pigServer.registerQuery("d = filter c by b0>b2;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("d");
+
+ Tuple t = iter.next();
+ assertTrue(t.toString().contains("(2,3,1,2)"));
+ assertFalse(iter.hasNext());
+ }
}