svn commit: r1556374 - in /pig/trunk: CHANGES.txt src/org/apache/pig/builtin/PigStorage.java test/org/apache/pig/test/TestPigStorage.java

[da...@apache.org]

Author: daijy
Date: Tue Jan  7 21:54:04 2014
New Revision: 1556374

URL: http://svn.apache.org/r1556374
Log:
PIG-3650: Fix for PIG-3100 breaks column pruning

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/builtin/PigStorage.java
    pig/trunk/test/org/apache/pig/test/TestPigStorage.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1556374&r1=1556373&r2=1556374&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan  7 21:54:04 2014
@@ -64,6 +64,8 @@ OPTIMIZATIONS
  
 BUG FIXES
 
+PIG-3650: Fix for PIG-3100 breaks column pruning (tmwoodruff via daijy)
+
 PIG-3643: Nested Foreach with UDF and bincond is broken (cheolsoo)
 
 PIG-3616: TestBuiltIn.testURIwithCurlyBrace() silently fails (lbendig via cheolsoo)

Modified: pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=1556374&r1=1556373&r2=1556374&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/PigStorage.java Tue Jan  7 21:54:04 2014
@@ -307,8 +307,12 @@ LoadPushDown, LoadMetadata, StoreMetadat
             // only contains required fields.
             // We walk the requiredColumns array to find required fields,
             // and cast those.
-            for (int i = 0; i < Math.min(fieldSchemas.length, tup.size()); i++) {
+            for (int i = 0; i < fieldSchemas.length; i++) {
                 if (mRequiredColumns == null || (mRequiredColumns.length>i && mRequiredColumns[i])) {
+                    if (tupleIdx >= tup.size()) {
+                        tup.append(null);
+                    }
+                    
                     Object val = null;
                     if(tup.get(tupleIdx) != null){
                         byte[] bytes = ((DataByteArray) tup.get(tupleIdx)).get();
@@ -319,9 +323,6 @@ LoadPushDown, LoadMetadata, StoreMetadat
                     tupleIdx++;
                 }
             }
-            for (int i = tup.size(); i < fieldSchemas.length; i++) {
-                tup.append(null);
-            }
         }
         return tup;
     }

Modified: pig/trunk/test/org/apache/pig/test/TestPigStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStorage.java?rev=1556374&r1=1556373&r2=1556374&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStorage.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStorage.java Tue Jan  7 21:54:04 2014
@@ -20,11 +20,7 @@ package org.apache.pig.test;
 
 import static org.apache.pig.ExecType.MAPREDUCE;
 import static org.apache.pig.builtin.mock.Storage.tuple;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.junit.Assert.*;
 
 import java.io.File;
 import java.io.FileWriter;
@@ -259,6 +255,27 @@ public class TestPigStorage  {
     }
 
     @Test
+    public void testPruneColumnsWithSchema() throws Exception {
+        pigContext.connect();
+        String query = "a = LOAD '" + datadir + "originput' using PigStorage(',') " +
+        "as (f1:chararray, f2:int);";
+        pig.registerQuery(query);
+        pig.store("a", datadir + "aout", "PigStorage('\\t', '-schema')");
+    
+        // aout now has a schema.
+    
+        // Verify that loaded data has the correct data type after the prune
+        pig.registerQuery("b = LOAD '" + datadir + "aout' using PigStorage('\\t'); c = FOREACH b GENERATE f2;");
+        
+        Iterator<Tuple> it = pig.openIterator("c");
+        Assert.assertTrue("results were produced", it.hasNext());
+        
+        Tuple t = it.next();
+        
+        Assert.assertTrue("data is correct type", t.get(0) instanceof Integer);
+    }
+
+    @Test
     public void testSchemaConversion() throws Exception {
 
         Util.createLocalInputFile(datadir + "originput2",
@@ -644,5 +661,12 @@ public class TestPigStorage  {
         assertTrue(it.hasNext());
         assertEquals(tuple(1,null,null), it.next());
         assertFalse(it.hasNext());
+        
+        // Now, test with prune
+        pig.registerQuery("a = load '"+Util.encodeEscape(inputDir.getAbsolutePath())+"'; b = foreach a generate y, z;");
+        it = pig.openIterator("b");
+        assertTrue(it.hasNext());
+        assertEquals(tuple(null,null), it.next());
+        assertFalse(it.hasNext());
     }
 }