You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2007/02/16 23:30:48 UTC
svn commit: r508625 - in /lucene/hadoop/trunk: CHANGES.txt
conf/hadoop-default.xml src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
src/java/org/apache/hadoop/mapred/MapOutputLocation.java
Author: cutting
Date: Fri Feb 16 14:30:46 2007
New Revision: 508625
URL: http://svn.apache.org/viewvc?view=rev&rev=508625
Log:
HADOOP-947. Disable in-memory merging during shuffle, as this is causing data corruption. Contributed by Devaraj.
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/conf/hadoop-default.xml
lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapOutputLocation.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=508625&r1=508624&r2=508625
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Fri Feb 16 14:30:46 2007
@@ -75,10 +75,13 @@
(Dhruba Borthakur via cutting)
-Branch 0.11 (unreleased)
+Release 0.11.2 - 2007-02-16
1. HADOOP-1009. Fix an infinite loop in the HDFS namenode.
(Dhruba Borthakur via cutting)
+
+ 2. HADOOP-947. Disable in-memory merging during shuffle, as this is
+ causing data corruption. (Devaraj Das via cutting)
Release 0.11.1 - 2007-02-09
Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?view=diff&rev=508625&r1=508624&r2=508625
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Fri Feb 16 14:30:46 2007
@@ -132,7 +132,7 @@
<property>
<name>fs.inmemory.size.mb</name>
- <value>75</value>
+ <value>0</value>
<description>The size of the in-memory filsystem instance in MB</description>
</property>
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java?view=diff&rev=508625&r1=508624&r2=508625
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/fs/InMemoryFileSystem.java Fri Feb 16 14:30:46 2007
@@ -386,7 +386,9 @@
}
public float getPercentUsed() {
- return (float)totalUsed/fsSize;
+ if (fsSize > 0)
+ return (float)totalUsed/fsSize;
+ else return 0.1f;
}
private boolean canFitInMemory(int size) {
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapOutputLocation.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapOutputLocation.java?view=diff&rev=508625&r1=508624&r2=508625
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapOutputLocation.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/MapOutputLocation.java Fri Feb 16 14:30:46 2007
@@ -208,8 +208,9 @@
int inMemFSSize = inMemFileSys.getFSSize();
int checksumLength = inMemFileSys.getChecksumFileLength(length);
- boolean createInMem =
- (((float)(length + checksumLength) / inMemFSSize <=
+ boolean createInMem = false;
+ if (inMemFSSize > 0)
+ createInMem = (((float)(length + checksumLength) / inMemFSSize <=
MAX_INMEM_FILESIZE_FRACTION) &&
inMemFileSys.reserveSpaceWithCheckSum(localFilename, length));