You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cd...@apache.org on 2008/06/03 05:57:17 UTC
svn commit: r662631 - in /hadoop/core/trunk: CHANGES.txt
src/java/org/apache/hadoop/mapred/MapTask.java
Author: cdouglas
Date: Mon Jun 2 20:57:17 2008
New Revision: 662631
URL: http://svn.apache.org/viewvc?rev=662631&view=rev
Log:
HADOOP-3475. Fix MapTask to correctly size the accounting allocation of io.sort.mb.
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=662631&r1=662630&r2=662631&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon Jun 2 20:57:17 2008
@@ -388,6 +388,9 @@
HADOOP-3471. Fix spurious errors from TestIndexedSort and add additional
logging to let failures be reproducible. (cdouglas)
+ HADOOP-3475. Fix MapTask to correctly size the accounting allocation of
+ io.sort.mb. (cdouglas)
+
Release 0.17.0 - 2008-05-18
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java?rev=662631&r1=662630&r2=662631&view=diff
==============================================================================
--- hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java (original)
+++ hadoop/core/trunk/src/java/org/apache/hadoop/mapred/MapTask.java Mon Jun 2 20:57:17 2008
@@ -324,6 +324,8 @@
private static final int KEYSTART = 1; // key offset in acct
private static final int VALSTART = 2; // val offset in acct
private static final int ACCTSIZE = 3; // total #fields in acct
+ private static final int RECSIZE =
+ (ACCTSIZE + 1) * 4; // acct bytes per record
// spill accounting
private volatile int numSpills = 0;
@@ -367,12 +369,12 @@
// buffers and accounting
int maxMemUsage = sortmb << 20;
int recordCapacity = (int)(maxMemUsage * recper);
- recordCapacity += (recordCapacity >>> 2) % 4;
+ recordCapacity -= recordCapacity % RECSIZE;
kvbuffer = new byte[maxMemUsage - recordCapacity];
bufvoid = kvbuffer.length;
- int kvcapacity = recordCapacity >>> 2;
- kvoffsets = new int[kvcapacity];
- kvindices = new int[recordCapacity - kvcapacity];
+ recordCapacity /= RECSIZE;
+ kvoffsets = new int[recordCapacity];
+ kvindices = new int[recordCapacity * ACCTSIZE];
softBufferLimit = (int)(kvbuffer.length * spillper);
softRecordLimit = (int)(kvoffsets.length * spillper);
// k/v serialization