You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2023/01/17 01:00:33 UTC
svn commit: r1906719 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java
Author: rohini
Date: Tue Jan 17 01:00:33 2023
New Revision: 1906719
URL: http://svn.apache.org/viewvc?rev=1906719&view=rev
Log:
PIG-5439: pig.exec.reducers.max does not take effect for skewed join (vnarayanan7 via rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1906719&r1=1906718&r2=1906719&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 17 01:00:33 2023
@@ -120,6 +120,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5439: pig.exec.reducers.max does not take effect for skewed join (vnarayanan7 via rohini)
+
PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini)
PIG-5433: Fix test failures with TestHBaseStorage and htrace dependency (rohini)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java?rev=1906719&r1=1906718&r2=1906719&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/FindQuantilesTez.java Tue Jan 17 01:00:33 2023
@@ -84,7 +84,8 @@ public class FindQuantilesTez extends Fi
long estimatedInputSize = (long)((double)sampleSize/mySamples.size() * totalInputRows);
estimatedNumReducers = (int)Math.ceil((double)estimatedInputSize/bytesPerTask);
- estimatedNumReducers = Math.min(estimatedNumReducers, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+ int maxReducers = PigMapReduce.sJobConfInternal.get().getInt(InputSizeReducerEstimator.MAX_REDUCER_COUNT_PARAM, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+ estimatedNumReducers = Math.min(estimatedNumReducers, maxReducers);
if (estimatedNumReducers==0) {
estimatedNumReducers = 1;
}
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java?rev=1906719&r1=1906718&r2=1906719&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/plan/udf/PartitionSkewedKeysTez.java Tue Jan 17 01:00:33 2023
@@ -70,7 +70,8 @@ public class PartitionSkewedKeysTez exte
InputSizeReducerEstimator.DEFAULT_BYTES_PER_REDUCER);
estimatedNumReducers = (int)Math.ceil((double)estimatedInputSize/bytesPerTask);
- estimatedNumReducers = Math.min(estimatedNumReducers, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+ int maxReducers = PigMapReduce.sJobConfInternal.get().getInt(InputSizeReducerEstimator.MAX_REDUCER_COUNT_PARAM, InputSizeReducerEstimator.DEFAULT_MAX_REDUCER_COUNT_PARAM);
+ estimatedNumReducers = Math.min(estimatedNumReducers, maxReducers);
LOG.info("Estimating parallelism: estimatedInputSize is " + estimatedInputSize + ". bytesPerTask is " + bytesPerTask + ". estimatedNumReducers is " + estimatedNumReducers + ".");