You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/07/19 19:11:00 UTC
svn commit: r965559 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/impl/logicalLayer/ColumnPruner.java
test/org/apache/pig/test/TestPruneColumn.java
Author: daijy
Date: Mon Jul 19 17:11:00 2010
New Revision: 965559
URL: http://svn.apache.org/viewvc?rev=965559&view=rev
Log:
PIG-1493: Column Pruner throw exception "inconsistent pruning"
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ColumnPruner.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=965559&r1=965558&r2=965559&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Jul 19 17:11:00 2010
@@ -341,6 +341,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-1493: Column Pruner throw exception "inconsistent pruning" (daijy)
+
PIG-1484: BinStorage should support comma seperated path (daijy)
PIG-1443: DefaultTuple underestimate the memory footprint for string (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ColumnPruner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ColumnPruner.java?rev=965559&r1=965558&r2=965559&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ColumnPruner.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/ColumnPruner.java Mon Jul 19 17:11:00 2010
@@ -134,19 +134,11 @@ public class ColumnPruner extends LOVisi
break;
for (Pair<Integer, Integer> relevantField: relevantFields.getFields())
{
+ // If any of the input column is pruned, prune this output column
if (columnsPruned.contains(relevantField))
{
columnPruned = true;
- }
- else {
- // For union, inconsistent pruning is possible (See PIG-1146)
- // We shall allow inconsistent pruning for union, and the pruneColumns method
- // in LOUnion will handle this inconsistency
- if (!(lOp instanceof LOUnion) && columnPruned==true) {
- int errCode = 2185;
- String msg = "Column $"+i+" of "+lOp+" inconsistent pruning";
- throw new OptimizerException(msg, errCode, PigException.BUG);
- }
+ break;
}
}
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=965559&r1=965558&r2=965559&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Mon Jul 19 17:11:00 2010
@@ -1878,4 +1878,24 @@ public class TestPruneColumn extends Tes
"No map keys pruned for A"}));
}
+ // See PIG-1493
+ @Test
+ public void testInconsistentPruning() throws Exception {
+ pigServer.registerQuery("A = load '"+ Util.generateURI(tmpFile1.toString(), pigServer.getPigContext()) + "' AS (a0:chararray, a1:chararray, a2);");
+ pigServer.registerQuery("B = foreach A generate CONCAT(a0,a1) as b0, a0, a2;");
+ pigServer.registerQuery("C = foreach B generate a0, a2;");
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+
+ assertTrue(iter.hasNext());
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1,3)"));
+
+ assertTrue(iter.hasNext());
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,2)"));
+
+ assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1",
+ "No map keys pruned for A"}));
+ }
+
}