You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/09/26 21:25:50 UTC
svn commit: r699457 - in /incubator/pig/branches/types: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
test/org/apache/pig/test/TestMRCompiler.java
test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
Author: olga
Date: Fri Sep 26 12:25:50 2008
New Revision: 699457
URL: http://svn.apache.org/viewvc?rev=699457&view=rev
Log:
PIG-462: LIMIT N should create one output file with N rows
Modified:
incubator/pig/branches/types/CHANGES.txt
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
incubator/pig/branches/types/test/org/apache/pig/test/TestMRCompiler.java
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
Modified: incubator/pig/branches/types/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/CHANGES.txt?rev=699457&r1=699456&r2=699457&view=diff
==============================================================================
--- incubator/pig/branches/types/CHANGES.txt (original)
+++ incubator/pig/branches/types/CHANGES.txt Fri Sep 26 12:25:50 2008
@@ -250,3 +250,6 @@
PIG-458: integration with Hadoop 18 (olgan)
PIG-459: increased sleep time before checking for job progress
+
+ PIG-462: LIMIT N should create one output file with N rows (shravanmn via
+ olgan)
Modified: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=699457&r1=699456&r2=699457&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java (original)
+++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java Fri Sep 26 12:25:50 2008
@@ -696,7 +696,7 @@
if (mro.reducePlan.isEmpty())
{
simpleConnectMapToReduce(mro);
- mro.requestedParallelism = -1;
+ mro.requestedParallelism = 1;
POLimit pLimit2 = new POLimit(new OperatorKey(scope,nig.getNextNodeId(scope)));
pLimit2.setLimit(op.getLimit());
mro.reducePlan.addAsLeaf(pLimit2);
@@ -1301,7 +1301,7 @@
POStore st = getStore();
st.setSFile(oldSpec);
limitAdjustMROp.reducePlan.addAsLeaf(st);
- limitAdjustMROp.requestedParallelism = -1;
+ limitAdjustMROp.requestedParallelism = 1;
// If the operator we're following has global sort set, we
// need to indicate that this is a limit after a sort.
// This will assure that we get the right sort comparator
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestMRCompiler.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestMRCompiler.java?rev=699457&r1=699456&r2=699457&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestMRCompiler.java Fri Sep 26 12:25:50 2008
@@ -828,6 +828,8 @@
byte[] b = new byte[MAX_SIZE];
int len = fis.read(b);
goldenPlan = new String(b, 0, len);
+ if (goldenPlan.charAt(len-1) == '\n')
+ goldenPlan = goldenPlan.substring(0, len-1);
pp.explain(System.out);
System.out.println();
Modified: incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld?rev=699457&r1=699456&r2=699457&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/MRC17.gld Fri Sep 26 12:25:50 2008
@@ -1,4 +1,4 @@
-MapReduce(-1) - -174:
+MapReduce(1) - -174:
| Store(DummyFil:DummyLdr) - -7856319821130535798
| |
| |---Limit - -180
@@ -14,4 +14,4 @@
| |
| |---Limit - -7398260302074824818
| |
-| |---Load(DummyFil:DummyLdr) - -4188863770717253580
\ No newline at end of file
+| |---Load(DummyFil:DummyLdr) - -4188863770717253580