You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2011/07/19 01:46:18 UTC
svn commit: r1148104 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/newplan/logical/relational/LOUnion.java
test/org/apache/pig/parser/TestUnionOnSchemaSetter.java
test/org/apache/pig/test/TestEvalPipeline2.java
Author: daijy
Date: Mon Jul 18 23:46:17 2011
New Revision: 1148104
URL: http://svn.apache.org/viewvc?rev=1148104&view=rev
Log:
PIG-2159: New logical plan uses incorrect class for SUM causing for ClassCastException
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LOUnion.java
pig/branches/branch-0.9/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1148104&r1=1148103&r2=1148104&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Mon Jul 18 23:46:17 2011
@@ -192,6 +192,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2159: New logical plan uses incorrect class for SUM causing for ClassCastException (daijy)
+
PIG-1890: Fix piggybank unit test TestAvroStorage (kengoodhope via daijy)
PIG-2144: ClassCastException when using IsEmpty(DIFF()) (thejas)
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LOUnion.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LOUnion.java?rev=1148104&r1=1148103&r2=1148104&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LOUnion.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/relational/LOUnion.java Mon Jul 18 23:46:17 2011
@@ -97,30 +97,48 @@ public class LOUnion extends LogicalRela
if (mergedSchema == null)
return null;
}
-
- // Bring back cached uid if any; otherwise, cache uid generated
- for (int i=0;i<s0.size();i++)
- {
- LogicalSchema.LogicalFieldSchema fs = mergedSchema.getField(i);
- long uid = -1;
- for (Pair<Long, Long> pair : uidMapping) {
- if (pair.second==s0.getField(i).uid) {
- uid = pair.first;
- break;
- }
+ }
+
+ // Bring back cached uid if any; otherwise, cache uid generated
+ for (int i=0;i<s0.size();i++)
+ {
+ LogicalSchema.LogicalFieldSchema outputFieldSchema;
+ if (onSchema) {
+ outputFieldSchema = mergedSchema.getFieldSubNameMatch(s0.getField(i).alias);
+ } else {
+ outputFieldSchema = mergedSchema.getField(i);
+ }
+ long uid = -1;
+ for (Pair<Long, Long> pair : uidMapping) {
+ if (pair.second==s0.getField(i).uid) {
+ uid = pair.first;
+ break;
}
- if (uid==-1) {
- uid = LogicalExpression.getNextUid();
- for (Operator input : inputs) {
- long inputUid = ((LogicalRelationalOperator)input).getSchema().getField(i).uid;
- uidMapping.add(new Pair<Long, Long>(uid, inputUid));
+ }
+ if (uid==-1) {
+ uid = LogicalExpression.getNextUid();
+ for (Operator input : inputs) {
+ long inputUid;
+ LogicalFieldSchema matchedInputFieldSchema;
+ if (onSchema) {
+ matchedInputFieldSchema = ((LogicalRelationalOperator)input).getSchema().getFieldSubNameMatch(s0.getField(i).alias);
+ if (matchedInputFieldSchema!=null) {
+ inputUid = matchedInputFieldSchema.uid;
+ uidMapping.add(new Pair<Long, Long>(uid, inputUid));
+ }
}
+ else {
+ matchedInputFieldSchema = mergedSchema.getField(i);
+ inputUid = ((LogicalRelationalOperator)input).getSchema().getField(i).uid;
+ uidMapping.add(new Pair<Long, Long>(uid, inputUid));
+ }
+
}
-
- fs.uid = uid;
}
- }
+ outputFieldSchema.uid = uid;
+ }
+
return schema = mergedSchema;
}
Modified: pig/branches/branch-0.9/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java?rev=1148104&r1=1148103&r2=1148104&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/parser/TestUnionOnSchemaSetter.java Mon Jul 18 23:46:17 2011
@@ -24,7 +24,10 @@ import org.apache.pig.impl.logicalLayer.
import org.apache.pig.newplan.logical.relational.LogicalPlan;
import org.apache.pig.newplan.logical.visitor.UnionOnSchemaSetter;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+@RunWith(JUnit4.class)
public class TestUnionOnSchemaSetter {
@Test
public void test1() throws FrontendException {
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java?rev=1148104&r1=1148103&r2=1148104&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestEvalPipeline2.java Mon Jul 18 23:46:17 2011
@@ -1542,4 +1542,47 @@ public class TestEvalPipeline2 {
pigServer.getPigContext().getProperties().remove("pig.optimizer.rules");
}
+
+ // See PIG-2159
+ @Test
+ public void testUnionOnSchemaUidGeneration() throws Exception{
+ String[] input1 = {
+ "100,101,102,103,104,105",
+ "110,111,112,113,114,115"
+ };
+
+ String[] input2 = {
+ "200,201,202,203,204,205",
+ "210,211,212,213,214,215"
+ };
+
+ String[] input0 = {
+ "200,201,202,203,204,205",
+ "210,211,212,213,214,215"
+ };
+
+ Util.createInputFile(cluster, "table_testUnionOnSchemaUidGeneration1", input1);
+ Util.createInputFile(cluster, "table_testUnionOnSchemaUidGeneration2", input2);
+ Util.createInputFile(cluster, "table_testUnionOnSchemaUidGeneration0", input0);
+
+ pigServer.registerQuery("A = load 'table_testUnionOnSchemaUidGeneration1' using PigStorage(',') as (f1:int,f2:int,f3:int,f4:long,f5:double);");
+ pigServer.registerQuery("B = load 'table_testUnionOnSchemaUidGeneration2' using PigStorage(',') as (f1:int,f2:int,f3:int,f4:long,f5:double);");
+ pigServer.registerQuery("C = load 'table_testUnionOnSchemaUidGeneration0' using PigStorage(',') as (f1:int,f2:int,f3:int);");
+ pigServer.registerQuery("U = UNION ONSCHEMA A,B;");
+ pigServer.registerQuery("J = join C by (f1,f2,f3) LEFT OUTER, U by (f1,f2,f3);");
+ pigServer.registerQuery("Porj = foreach J generate C::f1 as f1 ,C::f2 as f2,C::f3 as f3,U::f4 as f4,U::f5 as f5;");
+ pigServer.registerQuery("G = GROUP Porj by (f1,f2,f3,f5);");
+ pigServer.registerQuery("Final = foreach G generate SUM(Porj.f4) as total;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("Final");
+
+ Tuple t = iter.next();
+ Assert.assertTrue(t.toString().equals("(203)"));
+
+ t = iter.next();
+ Assert.assertTrue(t.toString().equals("(213)"));
+
+ Assert.assertFalse(iter.hasNext());
+
+ }
}