You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2016/05/16 17:29:26 UTC
svn commit: r1744098 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java

Author: rohini
Date: Mon May 16 17:29:26 2016
New Revision: 1744098

URL: http://svn.apache.org/viewvc?rev=1744098&view=rev
Log:
PIG-4895: User UDFs relying on mapreduce.job.maps broken in Tez (rohini)

Modified:
    pig/trunk/CHANGES.txt
    pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java

Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1744098&r1=1744097&r2=1744098&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon May 16 17:29:26 2016
@@ -123,6 +123,8 @@ PIG-4639: Add better parser for Apache H
 
 BUG FIXES
 
+PIG-4895: User UDFs relying on mapreduce.job.maps broken in Tez (rohini)
+
 PIG-4883: MapKeyType of splitter was set wrongly in specific multiquery case (kellyzly via rohini)
 
 PIG-4887: Parameter substitution skipped with glob on register (knoguchi)

Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java?rev=1744098&r1=1744097&r2=1744098&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/runtime/PigProcessor.java Mon May 16 17:29:26 2016
@@ -41,6 +41,7 @@ import org.apache.pig.backend.executione
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRConfiguration;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigHadoopLogger;
+import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.ProgressableReporter;
 import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.UDFFinishVisitor;
@@ -158,6 +159,12 @@ public class PigProcessor extends Abstra
         conf.setInt(JobContext.TASK_PARTITION,
               taskAttemptId.getTaskID().getId());
         conf.set(JobContext.ID, taskAttemptId.getJobID().toString());
+        if (conf.get(PigInputFormat.PIG_INPUT_LIMITS) != null) {
+            // Has Load and is a root vertex
+            conf.setInt(JobContext.NUM_MAPS, getContext().getVertexParallelism());
+        } else {
+            conf.setInt(JobContext.NUM_REDUCES, getContext().getVertexParallelism());
+        }
 
         conf.set(PigConstants.TASK_INDEX, Integer.toString(getContext().getTaskIndex()));
         UDFContext.getUDFContext().addJobConf(conf);