You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by an...@apache.org on 2013/10/04 05:42:08 UTC
svn commit: r1529075 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Author: aniket486
Date: Fri Oct 4 03:42:07 2013
New Revision: 1529075
URL: http://svn.apache.org/r1529075
Log:
PIG-3497: JobControlCompiler should only do reducer estimation when the job has a reduce phase (amatsukawa via aniket486)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1529075&r1=1529074&r2=1529075&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Oct 4 03:42:07 2013
@@ -32,6 +32,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-3497: JobControlCompiler should only do reducer estimation when the job has a reduce phase (amatsukawa via aniket486)
+
PIG-3469: Skewed join can cause unrecoverable NullPointerException when one of its inputs is missing (Jarek Jarcec Cecho via xuefuz)
Release 0.12.0 (unreleased changes)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java?rev=1529075&r1=1529074&r2=1529075&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/JobControlCompiler.java Fri Oct 4 03:42:07 2013
@@ -479,8 +479,6 @@ public class JobControlCompiler{
}
}
- adjustNumReducers(plan, mro, nwJob);
-
if(lds!=null && lds.size()>0){
for (POLoad ld : lds) {
//Store the target operators for tuples read
@@ -658,6 +656,7 @@ public class JobControlCompiler{
POPackage pack = null;
if(mro.reducePlan.isEmpty()){
//MapOnly Job
+ log.info("Map only job, skipping reducer estimation");
nwJob.setMapperClass(PigMapOnly.Map.class);
nwJob.setNumReduceTasks(0);
if(!pigContext.inIllustrator)
@@ -671,6 +670,11 @@ public class JobControlCompiler{
}
else{
//Map Reduce Job
+
+ // Estimate the number of reducers
+ log.info("Reduce phase detected, estimating # of required reducers.");
+ adjustNumReducers(plan, mro, nwJob);
+
//Process the POPackage operator and remove it from the reduce plan
if(!mro.combinePlan.isEmpty()){
POPackage combPack = (POPackage)mro.combinePlan.getRoots().get(0);