You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by bu...@apache.org on 2007/04/17 22:08:42 UTC
svn commit: r529749 - in /lucene/java/trunk: CHANGES.txt
src/java/org/apache/lucene/store/RAMInputStream.java
src/java/org/apache/lucene/store/RAMOutputStream.java
src/test/org/apache/lucene/store/MockRAMOutputStream.java
Author: buschmi
Date: Tue Apr 17 13:08:41 2007
New Revision: 529749
URL: http://svn.apache.org/viewvc?view=rev&rev=529749
Log:
LUCENE-431: RAMInputStream and RAMOutputStream subclass IndexInput and IndexOutput directly now
Modified:
lucene/java/trunk/CHANGES.txt
lucene/java/trunk/src/java/org/apache/lucene/store/RAMInputStream.java
lucene/java/trunk/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java
Modified: lucene/java/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/CHANGES.txt?view=diff&rev=529749&r1=529748&r2=529749
==============================================================================
--- lucene/java/trunk/CHANGES.txt (original)
+++ lucene/java/trunk/CHANGES.txt Tue Apr 17 13:08:41 2007
@@ -130,6 +130,10 @@
of SegmentTermPositions instead of SegmentTermDocs without additional costs.
(Michael Busch)
+ 2. LUCENE-431: RAMInputStream and RAMOutputStream extend IndexInput and
+ IndexOutput directly now. This avoids further buffering and thus avoids
+ unneccessary array copies. (Michael Busch)
+
Documentation:
1. LUCENE 791 && INFRA-1173: Infrastructure moved the Wiki to http://wiki.apache.org/lucene-java/ Updated the links in the docs and wherever else I found references. (Grant Ingersoll, Joe Schaefer)
Modified: lucene/java/trunk/src/java/org/apache/lucene/store/RAMInputStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/RAMInputStream.java?view=diff&rev=529749&r1=529748&r2=529749
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/RAMInputStream.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/RAMInputStream.java Tue Apr 17 13:08:41 2007
@@ -1,5 +1,7 @@
package org.apache.lucene.store;
+import java.io.IOException;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -19,46 +21,90 @@
/**
* A memory-resident {@link IndexInput} implementation.
- *
+ *
* @version $Id$
*/
-class RAMInputStream extends BufferedIndexInput implements Cloneable {
+class RAMInputStream extends IndexInput implements Cloneable {
+ static final int BUFFER_SIZE = BufferedIndexOutput.BUFFER_SIZE;
+
private RAMFile file;
- private long pointer = 0;
private long length;
+ private byte[] currentBuffer;
+ private int currentBufferIndex;
+
+ private int bufferPosition;
+ private long bufferStart;
+ private int bufferLength;
+
public RAMInputStream(RAMFile f) {
file = f;
length = file.length;
+
+ // make sure that we switch to the
+ // first needed buffer lazily
+ currentBufferIndex = -1;
+ currentBuffer = null;
}
- public void readInternal(byte[] dest, int destOffset, int len) {
- int remainder = len;
- long start = pointer;
- while (remainder != 0) {
- int bufferNumber = (int)(start/BUFFER_SIZE);
- int bufferOffset = (int)(start%BUFFER_SIZE);
- int bytesInBuffer = BUFFER_SIZE - bufferOffset;
- int bytesToCopy = bytesInBuffer >= remainder ? remainder : bytesInBuffer;
- byte[] buffer = (byte[])file.buffers.get(bufferNumber);
- System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy);
- destOffset += bytesToCopy;
- start += bytesToCopy;
- remainder -= bytesToCopy;
+ public void close() {
+ // nothing to do here
+ }
+
+ public long length() {
+ return length;
+ }
+
+ public byte readByte() throws IOException {
+ if (bufferPosition >= bufferLength) {
+ currentBufferIndex++;
+ switchCurrentBuffer();
}
- pointer += len;
+ return currentBuffer[bufferPosition++];
}
- public void close() {
+ public void readBytes(byte[] b, int offset, int len) throws IOException {
+ while (len > 0) {
+ if (bufferPosition >= bufferLength) {
+ currentBufferIndex++;
+ switchCurrentBuffer();
+ }
+
+ int remainInBuffer = bufferLength - bufferPosition;
+ int bytesToCopy = len < remainInBuffer ? len : remainInBuffer;
+ System.arraycopy(currentBuffer, bufferPosition, b, offset, bytesToCopy);
+ offset += bytesToCopy;
+ len -= bytesToCopy;
+ bufferPosition += bytesToCopy;
+ }
}
- public void seekInternal(long pos) {
- pointer = pos;
+ private final void switchCurrentBuffer() throws IOException {
+ if (currentBufferIndex >= file.buffers.size()) {
+ // end of file reached, no more buffers left
+ throw new IOException("Read past EOF");
+ } else {
+ currentBuffer = (byte[]) file.buffers.get(currentBufferIndex);
+ bufferPosition = 0;
+ bufferStart = BUFFER_SIZE * currentBufferIndex;
+ bufferLength = (int) (length - bufferStart);
+ if (bufferLength > BUFFER_SIZE) {
+ bufferLength = BUFFER_SIZE;
+ }
+ }
}
- public long length() {
- return length;
+ public long getFilePointer() {
+ return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
}
+ public void seek(long pos) throws IOException {
+ long bufferStart = currentBufferIndex * BUFFER_SIZE;
+ if (pos < bufferStart || pos >= bufferStart + BUFFER_SIZE) {
+ currentBufferIndex = (int) (pos / BUFFER_SIZE);
+ switchCurrentBuffer();
+ }
+ bufferPosition = (int) (pos % BUFFER_SIZE);
+ }
}
Modified: lucene/java/trunk/src/java/org/apache/lucene/store/RAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/store/RAMOutputStream.java?view=diff&rev=529749&r1=529748&r2=529749
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/store/RAMOutputStream.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/store/RAMOutputStream.java Tue Apr 17 13:08:41 2007
@@ -21,13 +21,21 @@
/**
* A memory-resident {@link IndexOutput} implementation.
- *
+ *
* @version $Id$
*/
-public class RAMOutputStream extends BufferedIndexOutput {
+public class RAMOutputStream extends IndexOutput {
+ static final int BUFFER_SIZE = BufferedIndexOutput.BUFFER_SIZE;
+
private RAMFile file;
- private long pointer = 0;
+
+ private byte[] currentBuffer;
+ private int currentBufferIndex;
+
+ private int bufferPosition;
+ private long bufferStart;
+ private int bufferLength;
/** Construct an empty output buffer. */
public RAMOutputStream() {
@@ -36,6 +44,11 @@
RAMOutputStream(RAMFile f) {
file = f;
+
+ // make sure that we switch to the
+ // first needed buffer lazily
+ currentBufferIndex = -1;
+ currentBuffer = null;
}
/** Copy the current contents of this buffer to the named output. */
@@ -66,41 +79,74 @@
file.setLength(0);
}
- public void flushBuffer(byte[] src, int offset, int len) throws IOException {
- byte[] buffer;
- int bufferPos = 0;
- while (bufferPos != len) {
- int bufferNumber = (int)(pointer/BUFFER_SIZE);
- int bufferOffset = (int)(pointer%BUFFER_SIZE);
- int bytesInBuffer = BUFFER_SIZE - bufferOffset;
- int remainInSrcBuffer = len - bufferPos;
- int bytesToCopy = bytesInBuffer >= remainInSrcBuffer ? remainInSrcBuffer : bytesInBuffer;
-
- if (bufferNumber == file.buffers.size())
- buffer = file.addBuffer(BUFFER_SIZE);
- else
- buffer = (byte[]) file.buffers.get(bufferNumber);
-
- System.arraycopy(src, offset + bufferPos, buffer, bufferOffset, bytesToCopy);
- bufferPos += bytesToCopy;
- pointer += bytesToCopy;
+ public void close() throws IOException {
+ flush();
+ }
+
+ public void seek(long pos) throws IOException {
+ // set the file length in case we seek back
+ // and flush() has not been called yet
+ setFileLength();
+ if (pos < bufferStart || pos >= bufferStart + bufferLength) {
+ currentBufferIndex = (int) (pos / BUFFER_SIZE);
+ switchCurrentBuffer();
}
- if (pointer > file.length)
- file.setLength(pointer);
+ bufferPosition = (int) (pos % BUFFER_SIZE);
+ }
- file.setLastModified(System.currentTimeMillis());
+ public long length() {
+ return file.length;
}
- public void close() throws IOException {
- super.close();
+ public void writeByte(byte b) throws IOException {
+ if (bufferPosition == bufferLength) {
+ currentBufferIndex++;
+ switchCurrentBuffer();
+ }
+ currentBuffer[bufferPosition++] = b;
}
- public void seek(long pos) throws IOException {
- super.seek(pos);
- pointer = pos;
+ public void writeBytes(byte[] b, int offset, int len) throws IOException {
+ while (len > 0) {
+ if (bufferPosition == bufferLength) {
+ currentBufferIndex++;
+ switchCurrentBuffer();
+ }
+
+ int remainInBuffer = currentBuffer.length - bufferPosition;
+ int bytesToCopy = len < remainInBuffer ? len : remainInBuffer;
+ System.arraycopy(b, offset, currentBuffer, bufferPosition, bytesToCopy);
+ offset += bytesToCopy;
+ len -= bytesToCopy;
+ bufferPosition += bytesToCopy;
+ }
}
- public long length() {
- return file.length;
+
+ private final void switchCurrentBuffer() throws IOException {
+ if (currentBufferIndex == file.buffers.size()) {
+ currentBuffer = file.addBuffer(BUFFER_SIZE);
+ } else {
+ currentBuffer = (byte[]) file.buffers.get(currentBufferIndex);
+ }
+ bufferPosition = 0;
+ bufferStart = BUFFER_SIZE * currentBufferIndex;
+ bufferLength = currentBuffer.length;
+ }
+
+ private void setFileLength() {
+ long pointer = bufferStart + bufferPosition;
+ if (pointer > file.length) {
+ file.setLength(pointer);
+ }
+ }
+
+ public void flush() throws IOException {
+ file.setLastModified(System.currentTimeMillis());
+ setFileLength();
+ }
+
+ public long getFilePointer() {
+ return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
}
}
Modified: lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java?view=diff&rev=529749&r1=529748&r2=529749
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/store/MockRAMOutputStream.java Tue Apr 17 13:08:41 2007
@@ -30,6 +30,8 @@
public class MockRAMOutputStream extends RAMOutputStream {
private MockRAMDirectory dir;
private boolean first=true;
+
+ byte[] singleByte = new byte[1];
/** Construct an empty output buffer. */
public MockRAMOutputStream(MockRAMDirectory dir, RAMFile f) {
@@ -48,7 +50,12 @@
}
}
- public void flushBuffer(byte[] src, int offset, int len) throws IOException {
+ public void writeByte(byte b) throws IOException {
+ singleByte[0] = b;
+ writeBytes(singleByte, 0, 1);
+ }
+
+ public void writeBytes(byte[] b, int offset, int len) throws IOException {
long freeSpace = dir.maxSize - dir.sizeInBytes();
long realUsage = 0;
@@ -63,14 +70,14 @@
if (dir.maxSize != 0 && freeSpace <= len) {
if (freeSpace > 0 && freeSpace < len) {
realUsage += freeSpace;
- super.flushBuffer(src, offset, (int) freeSpace);
+ super.writeBytes(b, offset, (int) freeSpace);
}
if (realUsage > dir.maxUsedSize) {
dir.maxUsedSize = realUsage;
}
throw new IOException("fake disk full at " + dir.getRecomputedActualSizeInBytes() + " bytes");
} else {
- super.flushBuffer(src, offset, len);
+ super.writeBytes(b, offset, len);
}
if (first) {