You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/01 08:20:03 UTC
svn commit: r1295439 - in /pig/branches/branch-0.9: CHANGES.txt
src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
test/findbugsExcludeFile.xml test/org/apache/pig/test/TestPruneColumn.java
Author: daijy
Date: Thu Mar 1 07:20:02 2012
New Revision: 1295439
URL: http://svn.apache.org/viewvc?rev=1295439&view=rev
Log:
PIG-2534: Pig generating infinite map outputs
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
pig/branches/branch-0.9/test/findbugsExcludeFile.xml
pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1295439&r1=1295438&r2=1295439&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Thu Mar 1 07:20:02 2012
@@ -22,6 +22,8 @@ Release 0.9.3 - Unreleased
BUG FIXES
+PIG-2534: Pig generating infinite map outputs (daijy)
+
PIG-2532: Registered classes fail deserialization in frontend (traviscrawford via julien)
PIG-2493: UNION causes casting issues (vivekp via daijy)
Modified: pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java?rev=1295439&r1=1295438&r2=1295439&view=diff
==============================================================================
--- pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/newplan/logical/rules/ColumnPruneHelper.java Thu Mar 1 07:20:02 2012
@@ -301,6 +301,9 @@ public class ColumnPruneHelper {
@Override
public void visit(LOStream stream) throws FrontendException {
+ // output is not used, setOutputUids is used to check if it has output schema
+ Set<Long> output = setOutputUids(stream);
+
// Every field is required
LogicalRelationalOperator pred = (LogicalRelationalOperator)plan.getPredecessors(stream).get(0);
@@ -547,6 +550,8 @@ public class ColumnPruneHelper {
}
@SuppressWarnings("unchecked")
+ // Get output uid from output schema. If output schema does not exist,
+ // throw exception
private Set<Long> setOutputUids(LogicalRelationalOperator op) throws FrontendException {
List<Operator> ll = plan.getSuccessors(op);
Modified: pig/branches/branch-0.9/test/findbugsExcludeFile.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/findbugsExcludeFile.xml?rev=1295439&r1=1295438&r2=1295439&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/findbugsExcludeFile.xml (original)
+++ pig/branches/branch-0.9/test/findbugsExcludeFile.xml Thu Mar 1 07:20:02 2012
@@ -437,4 +437,9 @@
<Class name = "org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapBase$IllustratorContext" />
<Bug pattern = "SIC_INNER_SHOULD_BE_STATIC" />
</Match>
+ <Match>
+ <Class name = "org.apache.pig.newplan.logical.rules.ColumnPruneHelper$ColumnDependencyVisitor" />
+ <Method name = "visit" />
+ <Bug pattern = "DLS_DEAD_LOCAL_STORE" />
+ </Match>
</FindBugsFilter>
Modified: pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java?rev=1295439&r1=1295438&r2=1295439&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java (original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestPruneColumn.java Thu Mar 1 07:20:02 2012
@@ -2058,4 +2058,36 @@ public class TestPruneColumn extends Tes
assertFalse(iter.hasNext());
}
+
+ // See PIG-2534
+ @Test
+ public void testStream2() throws Exception {
+ File input1 = File.createTempFile("tmp", "");
+ input1.delete();
+ File input2 = File.createTempFile("tmp", "");
+ input2.delete();
+
+ Util.createLocalInputFile(input1.getAbsolutePath(), new String[]
+ {"[key1#0,key2#5,key3#val3,key4#val4,key5#val5]"});
+ Util.createLocalInputFile(input2.getAbsolutePath(), new String[]
+ {"[key1#0,key2#5,key3#val3,key4#val4,key5#val5]"});
+
+ pigServer.registerQuery("event_serve = LOAD '" + input1.getAbsolutePath() +
+ "' AS (s, m, l);");
+ pigServer.registerQuery("cm_data_raw = LOAD '" + input2.getAbsolutePath() +
+ "' AS (s, m, l);");
+ pigServer.registerQuery("cm_serve = FOREACH cm_data_raw GENERATE s#'key3' AS f1, s#'key4' AS f2, s#'key5' AS f3 ;");
+ pigServer.registerQuery("cm_serve_lowercase = stream cm_serve through `tr [:upper:] [:lower:]`;");
+ pigServer.registerQuery("cm_serve_final = FOREACH cm_serve_lowercase GENERATE $0 AS cm_event_guid, $1 AS cm_receive_time, $2 AS cm_ctx_url;");
+ pigServer.registerQuery("event_serve_project = FOREACH event_serve GENERATE s#'key3' AS event_guid, s#'key4' AS receive_time;");
+ pigServer.registerQuery("event_serve_join = join cm_serve_final by (cm_event_guid), event_serve_project by (event_guid);");
+ Iterator<Tuple> iter = pigServer.openIterator("event_serve_join");
+
+ String[] expected = new String[] {"(val3,val4,val5,val3,val4)"};
+
+ Util.checkQueryOutputsAfterSortRecursive(iter, expected, org.apache.pig.newplan.logical.Util.translateSchema(pigServer.dumpSchema("event_serve_join")));
+
+ assertTrue(checkLogFileMessage(new String[]{"Map key required for event_serve: $0->[key4, key3]",
+ "Map key required for cm_data_raw: $0->[key4, key3, key5]"}));
+ }
}