svn commit: r793826 - in /lucene/java/trunk: CHANGES.txt src/java/org/apache/lucene/store/MMapDirectory.java

[us...@apache.org]

Author: uschindler
Date: Tue Jul 14 09:30:31 2009
New Revision: 793826

URL: http://svn.apache.org/viewvc?rev=793826&view=rev
Log:
LUCENE-1741: Make MMapDirectory.MAX_BBUF user configureable to support chunking the index files in smaller parts

Modified:
    lucene/java/trunk/CHANGES.txt
    lucene/java/trunk/src/java/org/apache/lucene/store/MMapDirectory.java

Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?rev=793826&r1=793825&r2=793826&view=diff
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Tue Jul 14 09:30:31 2009
@@ -546,6 +546,11 @@
     boolean logic, can be used within quote operators with this parser, ie: 
     "(jo* -john) smyth~". (Mark Harwood via Mark Miller)
     
+32. LUCENE-1741: User configureable maximum chunk size in MMapDirectory.
+    On 32 bit platforms, the address space can be very fragmented, so
+    one big ByteBuffer for the whole file may not fit into address space.
+    (Eks Dev via Uwe Schindler)
+    
 Optimizations
 
  1. LUCENE-1427: Fixed QueryWrapperFilter to not waste time computing

Modified: lucene/java/trunk/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/MMapDirectory.java?rev=793826&r1=793825&r2=793826&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/MMapDirectory.java Tue Jul 14 09:30:31 2009
@@ -30,6 +30,8 @@
 import java.security.PrivilegedActionException;
 import java.lang.reflect.Method;
 
+import org.apache.lucene.util.Constants;
+
 /** File-based {@link Directory} implementation that uses
  *  mmap for reading, and {@link
  *  SimpleFSDirectory.SimpleFSIndexOutput} for writing.
@@ -40,6 +42,10 @@
  * be sure your have plenty of virtual address space, e.g. by
  * using a 64 bit JRE, or a 32 bit JRE with indexes that are
  * guaranteed to fit within the address space.
+ * On 32 bit platforms also consult {@link #setMaxChunkSize}
+ * if you have problems with mmap failing because of fragmented
+ * address space. If you get an OutOfMemoryException, it is recommened
+ * to reduce the chunk size, until it works.
  *
  * <p>Due to <a href="http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4724038">
  * this bug</a> in Sun's JRE, MMapDirectory's {@link IndexInput#close}
@@ -92,6 +98,7 @@
   static final Object[] NO_PARAMS = new Object[0];
   
   private boolean useUnmapHack = false;
+  private int maxBBuf = Constants.JRE_IS_64BIT ? Integer.MAX_VALUE : (256*1024*1024);
   
   /**
    * <code>true</code>, if this platform supports unmapping mmaped files.
@@ -109,7 +116,7 @@
     }
     UNMAP_SUPPORTED = v;
   }
-
+  
   /**
    * This method enables the workaround for unmapping the buffers
    * from address space after closing {@link IndexInput}, that is
@@ -164,6 +171,31 @@
       }
     }
   }
+  
+  /**
+   * Sets the maximum chunk size (default is {@link Integer#MAX_VALUE} for
+   * 64 bit JVMs and 256 MiBytes for 32 bit JVMs) used for memory mapping.
+   * Especially on 32 bit platform, the address space can be very fragmented,
+   * so large index files cannot be mapped.
+   * Using a lower chunk size makes the directory implementation a little
+   * bit slower (as the correct chunk must be resolved on each seek)
+   * but the chance is higher that mmap does not fail. On 64 bit
+   * Java platforms, this parameter should always be {@link Integer#MAX_VALUE},
+   * as the adress space is big enough.
+   */
+  public void setMaxChunkSize(final int maxBBuf) {
+    if (maxBBuf<=0)
+      throw new IllegalArgumentException("Maximum chunk size for mmap must be >0");
+    this.maxBBuf=maxBBuf;
+  }
+  
+  /**
+   * Returns the current mmap chunk size.
+   * @see #setMaxChunkSize
+   */
+  public int getMaxChunkSize() {
+    return maxBBuf;
+  } 
 
   private class MMapIndexInput extends IndexInput {
 
@@ -357,17 +389,15 @@
     }
   }
   
-  private final int MAX_BBUF = Integer.MAX_VALUE;
-
   /** Creates an IndexInput for the file with the given name. */
   public IndexInput openInput(String name, int bufferSize) throws IOException {
     ensureOpen();
     File f =  new File(getFile(), name);
     RandomAccessFile raf = new RandomAccessFile(f, "r");
     try {
-      return (raf.length() <= MAX_BBUF)
+      return (raf.length() <= (long) maxBBuf)
              ? (IndexInput) new MMapIndexInput(raf)
-             : (IndexInput) new MultiMMapIndexInput(raf, MAX_BBUF);
+             : (IndexInput) new MultiMMapIndexInput(raf, maxBBuf);
     } finally {
       raf.close();
     }