You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@lucene.apache.org by cu...@apache.org on 2004/10/04 21:44:11 UTC
cvs commit: jakarta-lucene/src/java/org/apache/lucene/store MMapDirectory.java
cutting 2004/10/04 12:44:11
Modified: src/java/org/apache/lucene/store MMapDirectory.java
Log:
Improved version by Paul Elschot that can handle files longer than 2^31.
Revision Changes Path
1.3 +128 -12 jakarta-lucene/src/java/org/apache/lucene/store/MMapDirectory.java
Index: MMapDirectory.java
===================================================================
RCS file: /home/cvs/jakarta-lucene/src/java/org/apache/lucene/store/MMapDirectory.java,v
retrieving revision 1.2
retrieving revision 1.3
diff -u -r1.2 -r1.3
--- MMapDirectory.java 28 Sep 2004 21:55:59 -0000 1.2
+++ MMapDirectory.java 4 Oct 2004 19:44:11 -0000 1.3
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
import java.io.IOException;
import java.io.File;
import java.io.RandomAccessFile;
@@ -29,24 +29,17 @@
* org.apache.lucene.FSDirectory.class set to
* org.apache.lucene.store.MMapDirectory. This will cause {@link
* FSDirectory#getDirectory(File,boolean)} to return instances of this class.
- *
- * @author Doug Cutting
*/
public class MMapDirectory extends FSDirectory {
- private class MMapIndexInput extends IndexInput {
+ private static class MMapIndexInput extends IndexInput {
private ByteBuffer buffer;
private final long length;
- public MMapIndexInput(File file) throws IOException {
- RandomAccessFile raf = new RandomAccessFile(file, "r");
- try {
+ private MMapIndexInput(RandomAccessFile raf) throws IOException {
this.length = raf.length();
this.buffer = raf.getChannel().map(MapMode.READ_ONLY, 0, length);
- } finally {
- raf.close();
- }
}
public byte readByte() throws IOException {
@@ -77,11 +70,134 @@
}
public void close() throws IOException {}
+ }
+ /* Added class MultiMMapIndexInput, Paul Elschot.
+ * Slightly adapted constructor of MMapIndexInput.
+ * Licensed under the Apache License, Version 2.0.
+ */
+ private static class MultiMMapIndexInput extends IndexInput {
+
+ private ByteBuffer[] buffers;
+ private int[] bufSizes; // keep here, ByteBuffer.size() method is optional
+
+ private final long length;
+
+ private int curBufIndex;
+ private final int maxBufSize;
+
+ private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
+ private int curAvail; // redundant for speed: (bufSizes[curBufIndex] - curBuf.position())
+
+
+ public MultiMMapIndexInput(RandomAccessFile raf, int maxBufSize)
+ throws IOException {
+ this.length = raf.length();
+ this.maxBufSize = maxBufSize;
+
+ if (maxBufSize <= 0)
+ throw new IllegalArgumentException("Non positive maxBufSize: "
+ + maxBufSize);
+
+ if ((length / maxBufSize) > Integer.MAX_VALUE)
+ throw new IllegalArgumentException
+ ("RandomAccessFile too big for maximum buffer size: "
+ + raf.toString());
+
+ int nrBuffers = (int) (length / maxBufSize);
+ if ((nrBuffers * maxBufSize) < length) nrBuffers++;
+
+ this.buffers = new ByteBuffer[nrBuffers];
+ this.bufSizes = new int[nrBuffers];
+
+ long bufferStart = 0;
+ FileChannel rafc = raf.getChannel();
+ for (int bufNr = 0; bufNr < nrBuffers; bufNr++) {
+ int bufSize = (length > (bufferStart + maxBufSize))
+ ? maxBufSize
+ : (int) (length - bufferStart);
+ //System.out.println("mapping from: "+bufferStart+", size: "+bufSize);
+ this.buffers[bufNr] = rafc.map(MapMode.READ_ONLY,bufferStart,bufSize);
+ this.bufSizes[bufNr] = bufSize;
+ bufferStart += bufSize;
+ }
+ seek(0L);
+ }
+
+ public byte readByte() throws IOException {
+ // Performance might be improved by reading ahead into an array of
+ // eg. 128 bytes and readByte() from there.
+ if (curAvail == 0) {
+ curBufIndex++;
+ curBuf = buffers[curBufIndex];
+ curBuf.position(0); // index out of bounds when too many requested
+ curAvail = bufSizes[curBufIndex];
+ }
+ curAvail--;
+ return curBuf.get();
+ }
+
+ public void readBytes(byte[] b, int offset, int len) throws IOException {
+ while (len > curAvail) {
+ curBuf.get(b, offset, curAvail);
+ len -= curAvail;
+ offset += curAvail;
+ curBufIndex++;
+ curBuf = buffers[curBufIndex];
+ curBuf.position(0); // index out of bounds when too many requested
+ curAvail = bufSizes[curBufIndex];
+ }
+ curBuf.get(b, offset, len);
+ curAvail -= len;
+ }
+
+ public long getFilePointer() {
+ return (curBufIndex * (long) maxBufSize) + curBuf.position();
+ }
+
+ public void seek(long pos) throws IOException {
+ curBufIndex = (int) (pos / maxBufSize);
+ curBuf = buffers[curBufIndex];
+ int bufOffset = (int) (pos - (curBufIndex * maxBufSize));
+ curBuf.position(bufOffset);
+ curAvail = bufSizes[curBufIndex] - bufOffset;
+ }
+
+ public long length() {
+ return length;
+ }
+
+ public Object clone() {
+ MultiMMapIndexInput clone = (MultiMMapIndexInput)super.clone();
+ clone.buffers = new ByteBuffer[buffers.length];
+ // No need to clone bufSizes.
+ // Since most clones will use only one buffer, duplicate() could also be
+ // done lazy in clones, eg. when adapting curBuf.
+ for (int bufNr = 0; bufNr < buffers.length; bufNr++) {
+ clone.buffers[bufNr] = buffers[bufNr].duplicate();
+ }
+ try {
+ clone.seek(getFilePointer());
+ } catch(IOException ioe) {
+ throw new RuntimeException(ioe);
+ };
+ return clone;
+ }
+
+ public void close() throws IOException {}
}
+
+ private final int MAX_BBUF = Integer.MAX_VALUE;
public IndexInput openInput(String name) throws IOException {
- return new MMapIndexInput(new File(getFile(), name));
+ File f = new File(getFile(), name);
+ RandomAccessFile raf = new RandomAccessFile(f, "r");
+ try {
+ return (raf.length() <= MAX_BBUF)
+ ? (IndexInput) new MMapIndexInput(raf)
+ : (IndexInput) new MultiMMapIndexInput(raf, MAX_BBUF);
+ } finally {
+ raf.close();
+ }
}
}
-
---------------------------------------------------------------------
To unsubscribe, e-mail: lucene-dev-unsubscribe@jakarta.apache.org
For additional commands, e-mail: lucene-dev-help@jakarta.apache.org