You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2014/11/21 00:37:08 UTC
hadoop git commit: MAPREDUCE-6169. MergeQueue should release
reference to the current item from key and value at the end of the iteration
to save memory. (Zhihai Xu via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk eb4045e76 -> 90194ca1c
MAPREDUCE-6169. MergeQueue should release reference to the current item from key and value at the end of the iteration to save memory. (Zhihai Xu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/90194ca1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/90194ca1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/90194ca1
Branch: refs/heads/trunk
Commit: 90194ca1cbd695d48c3705121c2ac9a8554578a2
Parents: eb4045e
Author: Karthik Kambatla <ka...@apache.org>
Authored: Thu Nov 20 15:36:57 2014 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Nov 20 15:36:57 2014 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 ++++
.../src/main/java/org/apache/hadoop/mapred/Merger.java | 11 ++++++++++-
.../apache/hadoop/mapreduce/task/reduce/TestMerger.java | 2 ++
3 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90194ca1/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a58d602..7e8a882 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -237,6 +237,10 @@ Release 2.7.0 - UNRELEASED
OPTIMIZATIONS
+ MAPREDUCE-6169. MergeQueue should release reference to the current item
+ from key and value at the end of the iteration to save memory.
+ (Zhihai Xu via kasha)
+
BUG FIXES
MAPREDUCE-5918. LineRecordReader can return the same decompressor to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90194ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
index b44e742..fffa92a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Merger.java
@@ -528,9 +528,17 @@ public class Merger {
}
}
+ private void resetKeyValue() {
+ key = null;
+ value.reset(new byte[] {}, 0);
+ diskIFileValue.reset(new byte[] {}, 0);
+ }
+
public boolean next() throws IOException {
- if (size() == 0)
+ if (size() == 0) {
+ resetKeyValue();
return false;
+ }
if (minSegment != null) {
//minSegment is non-null for all invocations of next except the first
@@ -539,6 +547,7 @@ public class Merger {
adjustPriorityQueue(minSegment);
if (size() == 0) {
minSegment = null;
+ resetKeyValue();
return false;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/90194ca1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
index 651dd38..6e3bedf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapreduce/task/reduce/TestMerger.java
@@ -294,6 +294,8 @@ public class TestMerger {
// Now there should be no more input
Assert.assertFalse(mergeQueue.next());
Assert.assertEquals(1.0f, mergeQueue.getProgress().get(), epsilon);
+ Assert.assertTrue(mergeQueue.getKey() == null);
+ Assert.assertEquals(0, mergeQueue.getValue().getData().length);
}
private Progressable getReporter() {