You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/03/12 02:08:30 UTC
svn commit: r922097 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
test/org/apache/pig/test/TestPruneColumn.java
Author: daijy
Date: Fri Mar 12 01:08:30 2010
New Revision: 922097
URL: http://svn.apache.org/viewvc?rev=922097&view=rev
Log:
PIG-1272: Column pruner causes wrong results
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=922097&r1=922096&r2=922097&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Mar 12 01:08:30 2010
@@ -147,6 +147,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1272: Column pruner causes wrong results (daijy)
+
PIG-1275: empty bag in PigStorage read as null (daijy)
PIG-1252: Diamond splitter does not generate correct results when using
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java?rev=922097&r1=922096&r2=922097&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/optimizer/PruneColumns.java Fri Mar 12 01:08:30 2010
@@ -220,12 +220,25 @@ public class PruneColumns extends Logica
{
List<RequiredFields> requiredInputFieldsList = new ArrayList<RequiredFields>();
RequiredFields requiredFields = new RequiredFields(false);
- for (RequiredFields rf : requiredOutputInfo.requiredFieldsList)
+ for (int i=0;i<mPlan.getSuccessors(rlo).size();i++)
{
+ RequiredFields rf = null;
+ try {
+ rf = requiredOutputInfo.requiredFieldsList.get(i);
+ } catch (Exception e) {
+ }
if (rf!=null)
{
rf.reIndex(0);
requiredFields.merge(rf);
+ } else {
+ // need all fields
+ List<Pair<Integer, Integer>> l = new ArrayList<Pair<Integer, Integer>>();
+ for (int j=0;j<rlo.getSchema().size();j++)
+ l.add(new Pair<Integer, Integer>(0, j));
+ rf = new RequiredFields(l);
+ requiredFields.merge(rf);
+ break;
}
}
requiredInputFieldsList.add(requiredFields);
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=922097&r1=922096&r2=922097&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Fri Mar 12 01:08:30 2010
@@ -1844,5 +1844,25 @@ public class TestPruneColumn extends Tes
assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
"No map keys pruned for A", "[0,1,2]"}));
}
+
+ // See PIG-1272
+ @Test
+ public void testSplit4() throws Exception {
+ pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = foreach A generate a0;");
+ pigServer.registerQuery("C = join A by a0, B by a0;");
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1,2,3,1)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,5,2,2)"));
+
+ assertTrue(checkLogFileMessage(new String[]{"No column pruned for A",
+ "No map keys pruned for A"}));
+ }
}