You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/01 08:21:36 UTC
svn commit: r1295441 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
test/findbugsExcludeFile.xml test/org/apache/pig/test/TestPruneColumn.java
Author: daijy
Date: Thu Mar 1 07:21:35 2012
New Revision: 1295441
URL: http://svn.apache.org/viewvc?rev=1295441&view=rev
Log:
PIG-2534: Pig generating infinite map outputs
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
pig/trunk/test/findbugsExcludeFile.xml
pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1295441&r1=1295440&r2=1295441&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 1 07:21:35 2012
@@ -417,6 +417,8 @@ Release 0.9.3 - Unreleased
BUG FIXES
+PIG-2534: Pig generating infinite map outputs (daijy)
+
PIG-2493: UNION causes casting issues (vivekp via daijy)
PIG-2497: Order of execution of fs, store and sh commands in Pig is not maintained (daijy)
Modified: pig/trunk/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java?rev=1295441&r1=1295440&r2=1295441&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java Thu Mar 1 07:21:35 2012
@@ -316,6 +316,9 @@ public class ColumnPruneHelper {
@Override
public void visit(LOStream stream) throws FrontendException {
+ // output is not used, setOutputUids is used to check if it has output schema
+ Set<Long> output = setOutputUids(stream);
+
// Every field is required
LogicalRelationalOperator pred = (LogicalRelationalOperator)plan.getPredecessors(stream).get(0);
@@ -561,6 +564,8 @@ public class ColumnPruneHelper {
}
@SuppressWarnings("unchecked")
+ // Get output uid from output schema. If output schema does not exist,
+ // throw exception
private Set<Long> setOutputUids(LogicalRelationalOperator op) throws FrontendException {
List<Operator> ll = plan.getSuccessors(op);
Modified: pig/trunk/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/pig/trunk/test/findbugsExcludeFile.xml?rev=1295441&r1=1295440&r2=1295441&view=diff
==============================================================================
--- pig/trunk/test/findbugsExcludeFile.xml (original)
+++ pig/trunk/test/findbugsExcludeFile.xml Thu Mar 1 07:21:35 2012
@@ -445,4 +445,9 @@
<Class name = "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase$IllustratorContext" />
<Bug pattern = "SIC_INNER_SHOULD_BE_STATIC" />
</Match>
+ <Match>
+ <Class name = "org.apache.pig.newplan.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor" />
+ <Method name = "visit" />
+ <Bug pattern = "DLS_DEAD_LOCAL_STORE" />
+ </Match>
</FindBugsFilter>
Modified: pig/trunk/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPruneColumn.java?rev=1295441&r1=1295440&r2=1295441&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPruneColumn.java Thu Mar 1 07:21:35 2012
@@ -2071,4 +2071,35 @@ public class TestPruneColumn extends Tes
assertTrue(checkLogFileMessage(new String[]{"Columns pruned for A: $1"}));
}
+ // See PIG-2534
+ @Test
+ public void testStream2() throws Exception {
+ File input1 = File.createTempFile("tmp", "");
+ input1.delete();
+ File input2 = File.createTempFile("tmp", "");
+ input2.delete();
+
+ Util.createLocalInputFile(input1.getAbsolutePath(), new String[]
+ {"[key1#0,key2#5,key3#val3,key4#val4,key5#val5]"});
+ Util.createLocalInputFile(input2.getAbsolutePath(), new String[]
+ {"[key1#0,key2#5,key3#val3,key4#val4,key5#val5]"});
+
+ pigServer.registerQuery("event_serve = LOAD '" + input1.getAbsolutePath() +
+ "' AS (s, m, l);");
+ pigServer.registerQuery("cm_data_raw = LOAD '" + input2.getAbsolutePath() +
+ "' AS (s, m, l);");
+ pigServer.registerQuery("cm_serve = FOREACH cm_data_raw GENERATE s#'key3' AS f1, s#'key4' AS f2, s#'key5' AS f3 ;");
+ pigServer.registerQuery("cm_serve_lowercase = stream cm_serve through `tr [:upper:] [:lower:]`;");
+ pigServer.registerQuery("cm_serve_final = FOREACH cm_serve_lowercase GENERATE $0 AS cm_event_guid, $1 AS cm_receive_time, $2 AS cm_ctx_url;");
+ pigServer.registerQuery("event_serve_project = FOREACH event_serve GENERATE s#'key3' AS event_guid, s#'key4' AS receive_time;");
+ pigServer.registerQuery("event_serve_join = join cm_serve_final by (cm_event_guid), event_serve_project by (event_guid);");
+ Iterator<Tuple> iter = pigServer.openIterator("event_serve_join");
+
+ String[] expected = new String[] {"(val3,val4,val5,val3,val4)"};
+
+ Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("event_serve_join")));
+
+ assertTrue(checkLogFileMessage(new String[]{"Map key required for event_serve: $0->[key4, key3]",
+ "Map key required for cm_data_raw: $0->[key4, key3, key5]"}));
+ }
}