You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by oz...@apache.org on 2016/06/23 00:20:59 UTC
hadoop git commit: MAPREDUCE-6721.
mapreduce.reduce.shuffle.memory.limit.percent=0.0 should be legal to enforce
shuffle to disk. (Gera Shegalov via ozawa)
Repository: hadoop
Updated Branches:
refs/heads/trunk 17eae9ebb -> 79a728916
MAPREDUCE-6721. mapreduce.reduce.shuffle.memory.limit.percent=0.0 should be legal to enforce shuffle to disk. (Gera Shegalov via ozawa)
This closes #102
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79a72891
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79a72891
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79a72891
Branch: refs/heads/trunk
Commit: 79a7289165510072c46779251ebb010248cb0ce8
Parents: 17eae9e
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Jun 22 17:13:50 2016 -0700
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Jun 22 17:20:13 2016 -0700
----------------------------------------------------------------------
.../mapreduce/task/reduce/MergeManagerImpl.java | 2 +-
.../src/main/resources/mapred-default.xml | 3 ++-
.../mapreduce/task/reduce/TestMergeManager.java | 18 ++++++++++++++++++
3 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79a72891/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index c99a330..1673ff8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -178,7 +178,7 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
final float singleShuffleMemoryLimitPercent =
jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT,
DEFAULT_SHUFFLE_MEMORY_LIMIT_PERCENT);
- if (singleShuffleMemoryLimitPercent <= 0.0f
+ if (singleShuffleMemoryLimitPercent < 0.0f
|| singleShuffleMemoryLimitPercent > 1.0f) {
throw new IllegalArgumentException("Invalid value for "
+ MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT + ": "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79a72891/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index d973bd4..ebc43aa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -447,7 +447,8 @@
<name>mapreduce.reduce.shuffle.memory.limit.percent</name>
<value>0.25</value>
<description>Expert: Maximum percentage of the in-memory limit that a
- single shuffle can consume</description>
+ single shuffle can consume. Range of valid values is [0.0, 1.0]. If the value
+ is 0.0 map outputs are shuffled directly to disk.</description>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79a72891/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
index ef860af..1c0d25b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MROutputFiles;
import org.apache.hadoop.mapred.MapOutputFile;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.task.reduce.MergeManagerImpl.CompressAwarePath;
import org.junit.Assert;
import org.junit.Test;
@@ -289,4 +290,21 @@ public class TestMergeManager {
assertTrue("Large in-memory reduce area unusable: " + maxInMemReduce,
maxInMemReduce > Integer.MAX_VALUE);
}
+
+ @Test
+ public void testZeroShuffleMemoryLimitPercent() throws Exception {
+ final JobConf jobConf = new JobConf();
+ jobConf.setFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT, 0.0f);
+ final MergeManager<Text, Text> mgr =
+ new MergeManagerImpl<>(null, jobConf, mock(LocalFileSystem.class),
+ null, null, null, null, null, null, null, null, null, null,
+ new MROutputFiles());
+ final long mapOutputSize = 10;
+ final int fetcher = 1;
+ final MapOutput<Text, Text> mapOutput = mgr.reserve(
+ TaskAttemptID.forName("attempt_0_1_m_1_1"),
+ mapOutputSize, fetcher);
+ assertEquals("Tiny map outputs should be shuffled to disk", "DISK",
+ mapOutput.getDescription());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org