You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2016/11/22 21:34:00 UTC
[43/50] [abbrv] hadoop git commit: MAPREDUCE-6793. io.sort.factor
code default and mapred-default.xml values inconsistent. Contributed by Gera
Shegalov.
MAPREDUCE-6793. io.sort.factor code default and mapred-default.xml values inconsistent. Contributed by Gera Shegalov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/683e0c71
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/683e0c71
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/683e0c71
Branch: refs/heads/HADOOP-13070
Commit: 683e0c71fe09600a24bdd7b707a613fe70ff1f6e
Parents: 009452b
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Mon Nov 21 22:10:57 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Mon Nov 21 22:10:57 2016 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/mapred/MapTask.java | 6 ++++--
.../main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 4 ++++
.../apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java | 3 ++-
.../apache/hadoop/mapreduce/task/reduce/TestMergeManager.java | 7 +++++++
4 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index 45431e6..3753fba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -967,7 +967,8 @@ public class MapTask extends Task {
//sanity checks
final float spillper =
job.getFloat(JobContext.MAP_SORT_SPILL_PERCENT, (float)0.8);
- final int sortmb = job.getInt(JobContext.IO_SORT_MB, 100);
+ final int sortmb = job.getInt(MRJobConfig.IO_SORT_MB,
+ MRJobConfig.DEFAULT_IO_SORT_MB);
indexCacheMemoryLimit = job.getInt(JobContext.INDEX_CACHE_MEMORY_LIMIT,
INDEX_CACHE_MEMORY_LIMIT_DEFAULT);
if (spillper > (float)1.0 || spillper <= (float)0.0) {
@@ -1920,7 +1921,8 @@ public class MapTask extends Task {
}
}
- int mergeFactor = job.getInt(JobContext.IO_SORT_FACTOR, 100);
+ int mergeFactor = job.getInt(MRJobConfig.IO_SORT_FACTOR,
+ MRJobConfig.DEFAULT_IO_SORT_FACTOR);
// sort the segments only if there are intermediate merges
boolean sortSegments = segmentList.size() > mergeFactor;
//merge
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 1325b74..2ad86ef 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -215,8 +215,12 @@ public interface MRJobConfig {
public static final String IO_SORT_FACTOR = "mapreduce.task.io.sort.factor";
+ public static final int DEFAULT_IO_SORT_FACTOR = 10;
+
public static final String IO_SORT_MB = "mapreduce.task.io.sort.mb";
+ public static final int DEFAULT_IO_SORT_MB = 100;
+
public static final String INDEX_CACHE_MEMORY_LIMIT = "mapreduce.task.index.cache.limit.bytes";
public static final String PRESERVE_FAILED_TASK_FILES = "mapreduce.task.files.preserve.failedtasks";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
index 09fe0cb..f26c10a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/MergeManagerImpl.java
@@ -175,7 +175,8 @@ public class MergeManagerImpl<K, V> implements MergeManager<K, V> {
MRJobConfig.REDUCE_MEMORY_TOTAL_BYTES,
Runtime.getRuntime().maxMemory()) * maxInMemCopyUse);
- this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100);
+ this.ioSortFactor = jobConf.getInt(MRJobConfig.IO_SORT_FACTOR,
+ MRJobConfig.DEFAULT_IO_SORT_FACTOR);
final float singleShuffleMemoryLimitPercent =
jobConf.getFloat(MRJobConfig.SHUFFLE_MEMORY_LIMIT_PERCENT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/683e0c71/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
index 325d2f9..1e0dddd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMergeManager.java
@@ -207,6 +207,13 @@ public class TestMergeManager {
}
}
+ @Test
+ public void testIoSortDefaults() {
+ final JobConf jobConf = new JobConf();
+ assertEquals(10, jobConf.getInt(MRJobConfig.IO_SORT_FACTOR, 100));
+ assertEquals(100, jobConf.getInt(MRJobConfig.IO_SORT_MB, 10));
+ }
+
@SuppressWarnings({ "unchecked", "deprecation" })
@Test(timeout=10000)
public void testOnDiskMerger() throws IOException, URISyntaxException,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org