You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2013/10/11 23:51:06 UTC
svn commit: r1531438 - in /pig/branches/branch-0.12: CHANGES.txt
src/docs/src/documentation/content/xdocs/func.xml
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
test/org/apache/pig/test/TestJobSubmission.java
Author: daijy
Date: Fri Oct 11 21:51:06 2013
New Revision: 1531438
URL: http://svn.apache.org/r1531438
Log:
PIG-3512: Reducer estimater is broken by PIG-3497
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/src/docs/src/documentation/content/xdocs/func.xml
pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
pig/branches/branch-0.12/test/org/apache/pig/test/TestJobSubmission.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1531438&r1=1531437&r2=1531438&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Fri Oct 11 21:51:06 2013
@@ -18,6 +18,12 @@
Pig Change Log
+Release 0.12.1 (unreleased changes)
+
+BUG FIXES
+
+PIG-3512: Reducer estimater is broken by PIG-3497 (daijy)
+
Release 0.12.0
INCOMPATIBLE CHANGES
Modified: pig/branches/branch-0.12/src/docs/src/documentation/content/xdocs/func.xml
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/docs/src/documentation/content/xdocs/func.xml?rev=1531438&r1=1531437&r2=1531438&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/docs/src/documentation/content/xdocs/func.xml (original)
+++ pig/branches/branch-0.12/src/docs/src/documentation/content/xdocs/func.xml Fri Oct 11 21:51:06 2013
@@ -2629,7 +2629,7 @@ This function never increases the result
<p>x</p>
</td>
<td>
- <p>CEIL(x)</p>
+ <p>FLOOR(x)</p>
</td>
</tr>
Modified: pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1531438&r1=1531437&r2=1531438&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Oct 11 21:51:06 2013
@@ -478,6 +478,11 @@ public class JobControlCompiler{
inp.add(ld.getLFile());
}
}
+
+ if(!mro.reducePlan.isEmpty()){
+ log.info("Reduce phase detected, estimating # of required reducers.");
+ adjustNumReducers(plan, mro, nwJob);
+ }
if(lds!=null && lds.size()>0){
for (POLoad ld : lds) {
@@ -656,7 +661,6 @@ public class JobControlCompiler{
POPackage pack = null;
if(mro.reducePlan.isEmpty()){
//MapOnly Job
- log.info("Map only job, skipping reducer estimation");
nwJob.setMapperClass(PigMapOnly.Map.class);
nwJob.setNumReduceTasks(0);
if(!pigContext.inIllustrator)
@@ -670,11 +674,6 @@ public class JobControlCompiler{
}
else{
//Map Reduce Job
-
- // Estimate the number of reducers
- log.info("Reduce phase detected, estimating # of required reducers.");
- adjustNumReducers(plan, mro, nwJob);
-
//Process the POPackage operator and remove it from the reduce plan
if(!mro.combinePlan.isEmpty()){
POPackage combPack = (POPackage)mro.combinePlan.getRoots().get(0);
Modified: pig/branches/branch-0.12/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/test/TestJobSubmission.java?rev=1531438&r1=1531437&r2=1531438&view=diff
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ pig/branches/branch-0.12/test/org/apache/pig/test/TestJobSubmission.java Fri Oct 11 21:51:06 2013
@@ -698,7 +698,7 @@ public class TestJobSubmission {
Util.copyFromLocalToCluster(cluster, "test/org/apache/pig/test/data/passwd", ((POLoad) sort.mapPlan.getRoots().get(0)).getLFile().getFileName());
//First job is just foreach with projection, mapper-only job, so estimate gets ignored
- Util.assertParallelValues(-1, -1, reducer, 0, jobControl.getWaitingJobs().get(0).getJobConf());
+ Util.assertParallelValues(-1, -1, -1, 0, jobControl.getWaitingJobs().get(0).getJobConf());
jcc.updateMROpPlan(jobControl.getReadyJobs());
jobControl = jcc.compile(mrPlan, query);