You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2015/03/16 21:57:51 UTC
hadoop git commit: MAPREDUCE-6105. nconsistent configuration in
property mapreduce.reduce.shuffle.merge.percent. Contributed by Ray Chiang.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4568acaa6 -> 0e9f78dda
MAPREDUCE-6105. nconsistent configuration in property mapreduce.reduce.shuffle.merge.percent. Contributed by Ray Chiang.
(cherry picked from commit 26a23b11598b3757ed4973781890e2cd2fc270df)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e9f78dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e9f78dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e9f78dd
Branch: refs/heads/branch-2
Commit: 0e9f78dda268e4a7e3edc25b7cfc535e3f89a691
Parents: 4568aca
Author: Harsh J <ha...@cloudera.com>
Authored: Tue Mar 17 01:17:34 2015 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Tue Mar 17 02:27:33 2015 +0530
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 1 +
.../apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 5 +++--
3 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e9f78dd/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 5feb14d..c0c6d57 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -8,6 +8,9 @@ Release 2.8.0 - UNRELEASED
IMPROVEMENTS
+ MAPREDUCE-6105. Inconsistent configuration in property
+ mapreduce.reduce.shuffle.merge.percent. (Ray Chiang via harsh)
+
MAPREDUCE-4414. Add main methods to JobConf and YarnConfiguration,
for debug purposes. (Plamen Jeliazkov via harsh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e9f78dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index d445bb1..4cff68f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -300,6 +300,7 @@ public interface MRJobConfig {
= "mapreduce.reduce.shuffle.memory.limit.percent";
public static final String SHUFFLE_MERGE_PERCENT = "mapreduce.reduce.shuffle.merge.percent";
+ public static final float DEFAULT_SHUFFLE_MERGE_PERCENT = 0.66f;
public static final String REDUCE_FAILURES_MAXPERCENT = "mapreduce.reduce.failures.maxpercent";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e9f78dd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index a4b1aa8..8bf17ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -191,8 +191,9 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
this.memToMemMergeOutputsThreshold =
jobConf.getInt(MRJobConfig.REDUCE_MEMTOMEM_THRESHOLD, ioSortFactor);
this.mergeThreshold = (long)(this.memoryLimit *
- jobConf.getFloat(MRJobConfig.SHUFFLE_MERGE_PERCENT,
- 0.90f));
+ jobConf.getFloat(
+ MRJobConfig.SHUFFLE_MERGE_PERCENT,
+ MRJobConfig.DEFAULT_SHUFFLE_MERGE_PERCENT));
LOG.info("MergerManager: memoryLimit=" + memoryLimit + ", " +
"maxSingleShuffleLimit=" + maxSingleShuffleLimit + ", " +
"mergeThreshold=" + mergeThreshold + ", " +