You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by an...@apache.org on 2013/10/05 00:33:10 UTC
svn commit: r1529336 - in /pig/branches/branch-0.12: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Author: aniket486
Date: Fri Oct 4 22:33:10 2013
New Revision: 1529336
URL: http://svn.apache.org/r1529336
Log:
PIG-3497: JobControlCompiler should only do reducer estimation when the job has a reduce phase (amatsukawa via aniket486)
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1529336&r1=1529335&r2=1529336&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Fri Oct 4 22:33:10 2013
@@ -247,6 +247,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3497: JobControlCompiler should only do reducer estimation when the job has a reduce phase (amatsukawa via aniket486)
+
PIG-3495: Streaming udf e2e tests failures on Windows (daijy)
PIG-3494: Several fixes for e2e tests (daijy)
Modified: pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1529336&r1=1529335&r2=1529336&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Oct 4 22:33:10 2013
@@ -479,8 +479,6 @@ public class JobControlCompiler{
}
}
- adjustNumReducers(plan, mro, nwJob);
-
if(lds!=null && lds.size()>0){
for (POLoad ld : lds) {
//Store the target operators for tuples read
@@ -658,6 +656,7 @@ public class JobControlCompiler{
POPackage pack = null;
if(mro.reducePlan.isEmpty()){
//MapOnly Job
+ log.info("Map only job, skipping reducer estimation");
nwJob.setMapperClass(PigMapOnly.Map.class);
nwJob.setNumReduceTasks(0);
if(!pigContext.inIllustrator)
@@ -671,6 +670,11 @@ public class JobControlCompiler{
}
else{
//Map Reduce Job
+
+ // Estimate the number of reducers
+ log.info("Reduce phase detected, estimating # of required reducers.");
+ adjustNumReducers(plan, mro, nwJob);
+
//Process the POPackage operator and remove it from the reduce plan
if(!mro.combinePlan.isEmpty()){
POPackage combPack = (POPackage)mro.combinePlan.getRoots().get(0);