You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 04:36:37 UTC
svn commit: r1077060 -
/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Author: omalley
Date: Fri Mar 4 03:36:37 2011
New Revision: 1077060
URL: http://svn.apache.org/viewvc?rev=1077060&view=rev
Log:
commit 38b77ce8cd5005ca4634c77ff0b6e579bf73a2a8
Author: Chris Douglas <cd...@apache.org>
Date: Wed Nov 25 14:41:33 2009 -0800
MAPREDUCE:1182 from https://issues.apache.org/jira/secure/attachment/12424116/M1182-1v20.patch
+++ b/YAHOO-CHANGES.txt
+ MAPREDUCE-1182. Fix overflow in reduce causing allocations to exceed the
+ configured threshold. (cdouglas)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java?rev=1077060&r1=1077059&r2=1077060&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ReduceTask.java Fri Mar 4 03:36:37 2011
@@ -988,13 +988,13 @@ class ReduceTask extends Task {
* simultaneously after which a merge is triggered. */
private static final float MAX_STALLED_SHUFFLE_THREADS_FRACTION = 0.75f;
- private final int maxSize;
- private final int maxSingleShuffleLimit;
+ private final long maxSize;
+ private final long maxSingleShuffleLimit;
- private int size = 0;
+ private long size = 0;
private Object dataAvailable = new Object();
- private int fullSize = 0;
+ private long fullSize = 0;
private int numPendingRequests = 0;
private int numRequiredMapOutputs = 0;
private int numClosed = 0;
@@ -1007,10 +1007,10 @@ class ReduceTask extends Task {
throw new IOException("mapred.job.shuffle.input.buffer.percent" +
maxInMemCopyUse);
}
- maxSize = (int)Math.min(
+ maxSize = (long)Math.min(
Runtime.getRuntime().maxMemory() * maxInMemCopyUse,
Integer.MAX_VALUE);
- maxSingleShuffleLimit = (int)(maxSize * MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
+ maxSingleShuffleLimit = (long)(maxSize * MAX_SINGLE_SHUFFLE_SEGMENT_FRACTION);
LOG.info("ShuffleRamManager: MemoryLimit=" + maxSize +
", MaxSingleShuffleLimit=" + maxSingleShuffleLimit);
}
@@ -1122,10 +1122,6 @@ class ReduceTask extends Task {
return (float)fullSize/maxSize;
}
- int getMemoryLimit() {
- return maxSize;
- }
-
boolean canFitInMemory(long requestedSize) {
return (requestedSize < Integer.MAX_VALUE &&
requestedSize < maxSingleShuffleLimit);