You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2020/10/15 15:17:06 UTC
svn commit: r1882546 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
test/org/apache/pig/test/TestPlanGeneration.java
Author: knoguchi
Date: Thu Oct 15 15:17:06 2020
New Revision: 1882546
URL: http://svn.apache.org/viewvc?rev=1882546&view=rev
Log:
PIG-5404: FLATTEN infers wrong datatype (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1882546&r1=1882545&r2=1882546&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Oct 15 15:17:06 2020
@@ -100,6 +100,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5404: FLATTEN infers wrong datatype (knoguchi)
+
PIG-5243: describe with typecast on as-clause shows the types before the typecasting (knoguchi)
PIG-5403: streaming job stuck with script failure when combined with ORDER BY (knoguchi)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java?rev=1882546&r1=1882545&r2=1882546&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java Thu Oct 15 15:17:06 2020
@@ -200,12 +200,15 @@ public class ForEachUserSchemaVisitor ex
gen.setFlattenFlags(new boolean[index]);
if (needCast) {
// Insert the casterForEach into the plan and patch up the plan.
+ plan.add(casterForEach);
List <Operator> successorOps = plan.getSuccessors(foreach);
if (successorOps != null && successorOps.size() > 0){
- Operator next = plan.getSuccessors(foreach).get(0);
- plan.insertBetween(foreach, casterForEach, next);
- }else{
- plan.add(casterForEach);
+ // since successorOps will be updated as part of the inserts,
+ // creating a shallow-copy first before traversing the list
+ for (Operator next : successorOps.toArray(new Operator [successorOps.size()])) {
+ plan.insertBetween(foreach, casterForEach, next);
+ }
+ } else {
plan.connect(foreach,casterForEach);
}
Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1882546&r1=1882545&r2=1882546&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Thu Oct 15 15:17:06 2020
@@ -464,6 +464,27 @@ public class TestPlanGeneration {
}
@Test
+ public void testForEachWithCast8() throws Exception {
+ //ForEachWithCast + Split (PIG-5404)
+ String query = "A = load 'foo' as (a:int, b:int);\n" +
+ "B = foreach A generate a as a0:chararray, b as b:int;\n" +
+ "store B into 'output1';\n" +
+ "store B into 'output2';\n" ;
+
+ LogicalPlan lp = Util.parse(query, pc);
+ Util.optimizeNewLP(lp);
+
+ assertEquals("Two LOStores should be created", 2, lp.getSinks().size());
+ for(Operator leave : lp.getSinks() ) {
+ LOStore loStore = (LOStore) leave;
+ assertEquals("a0",loStore.getSchema().getField(0).alias);
+ assertEquals(DataType.CHARARRAY, loStore.getSchema().getField(0).type);
+ assertEquals("b", loStore.getSchema().getField(1).alias);
+ assertEquals(DataType.INTEGER, loStore.getSchema().getField(1).type);
+ }
+ }
+
+ @Test
// See PIG-2315
public void testAsType1() throws Exception {
Data data = Storage.resetData(ps);