You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/06/14 00:09:13 UTC
svn commit: r1602531 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/store/
Author: rmuir
Date: Fri Jun 13 22:09:12 2014
New Revision: 1602531
URL: http://svn.apache.org/r1602531
Log:
LUCENE-5760: Speed up BufferedIndexInput.randomAccessSlice
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1602531&r1=1602530&r2=1602531&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Jun 13 22:09:12 2014
@@ -33,7 +33,7 @@ New Features
filtered and constant-score queries to postings highlighter.
(Luca Cavanna via Robert Muir)
-* LUCENE-5731: Add RandomAccessInput, a random access API for directory.
+* LUCENE-5731, LUCENE-5760: Add RandomAccessInput, a random access API for directory.
Add DirectReader/Writer, optimized for reading packed integers directly
from Directory. Add Lucene49Codec and Lucene49DocValuesFormat that make
use of these. (Robert Muir)
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java?rev=1602531&r1=1602530&r2=1602531&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java Fri Jun 13 22:09:12 2014
@@ -241,7 +241,7 @@ public abstract class MultiLevelSkipList
// clone this stream, it is already at the start of the current level
skipStream[i] = skipStream[0].clone();
if (inputIsBuffered && length < BufferedIndexInput.BUFFER_SIZE) {
- ((BufferedIndexInput) skipStream[i]).setBufferSize((int) length);
+ ((BufferedIndexInput) skipStream[i]).setBufferSize(Math.max(BufferedIndexInput.MIN_BUFFER_SIZE, (int) length));
}
// move base stream beyond the current level
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java?rev=1602531&r1=1602530&r2=1602531&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java Fri Jun 13 22:09:12 2014
@@ -21,11 +21,14 @@ import java.io.EOFException;
import java.io.IOException;
/** Base implementation class for buffered {@link IndexInput}. */
-public abstract class BufferedIndexInput extends IndexInput {
+public abstract class BufferedIndexInput extends IndexInput implements RandomAccessInput {
/** Default buffer size set to {@value #BUFFER_SIZE}. */
public static final int BUFFER_SIZE = 1024;
+ /** Minimum buffer size allowed */
+ public static final int MIN_BUFFER_SIZE = 8;
+
// The normal read buffer size defaults to 1024, but
// increasing this during merging seems to yield
// performance gains. However we don't want to increase
@@ -104,8 +107,8 @@ public abstract class BufferedIndexInput
}
private void checkBufferSize(int bufferSize) {
- if (bufferSize <= 0)
- throw new IllegalArgumentException("bufferSize must be greater than 0 (got " + bufferSize + ")");
+ if (bufferSize <= MIN_BUFFER_SIZE)
+ throw new IllegalArgumentException("bufferSize must be greater than MIN_BUFFER_SIZE (got " + bufferSize + ")");
}
@Override
@@ -255,6 +258,74 @@ public abstract class BufferedIndexInput
}
}
+ @Override
+ public final byte readByte(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ return buffer[(int)index];
+ }
+
+ @Override
+ public final short readShort(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength-1) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ return (short) (((buffer[(int)index] & 0xFF) << 8) |
+ (buffer[(int)index+1] & 0xFF));
+ }
+
+ @Override
+ public final int readInt(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength-3) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ return ((buffer[(int)index] & 0xFF) << 24) |
+ ((buffer[(int)index+1] & 0xFF) << 16) |
+ ((buffer[(int)index+2] & 0xFF) << 8) |
+ (buffer[(int)index+3] & 0xFF);
+ }
+
+ @Override
+ public final long readLong(long pos) throws IOException {
+ long index = pos - bufferStart;
+ if (index < 0 || index >= bufferLength-7) {
+ bufferStart = pos;
+ bufferPosition = 0;
+ bufferLength = 0; // trigger refill() on read()
+ seekInternal(pos);
+ refill();
+ index = 0;
+ }
+ final int i1 = ((buffer[(int)index] & 0xFF) << 24) |
+ ((buffer[(int)index+1] & 0xFF) << 16) |
+ ((buffer[(int)index+2] & 0xFF) << 8) |
+ (buffer[(int)index+3] & 0xFF);
+ final int i2 = ((buffer[(int)index+4] & 0xFF) << 24) |
+ ((buffer[(int)index+5] & 0xFF) << 16) |
+ ((buffer[(int)index+6] & 0xFF) << 8) |
+ (buffer[(int)index+7] & 0xFF);
+ return (((long)i1) << 32) | (i2 & 0xFFFFFFFFL);
+ }
+
private void refill() throws IOException {
long start = bufferStart + bufferPosition;
long end = start + bufferSize;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java?rev=1602531&r1=1602530&r2=1602531&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/ByteBufferIndexInput.java Fri Jun 13 22:09:12 2014
@@ -277,12 +277,6 @@ abstract class ByteBufferIndexInput exte
return buildSlice(sliceDescription, offset, length);
}
-
- @Override
- public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
- // note: technically we could even avoid the clone...
- return slice(null, offset, length);
- }
/** Builds the actual sliced IndexInput (may apply extra offset in subclasses). **/
protected ByteBufferIndexInput buildSlice(String sliceDescription, long offset, long length) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/IndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/IndexInput.java?rev=1602531&r1=1602530&r2=1602531&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/IndexInput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/store/IndexInput.java Fri Jun 13 22:09:12 2014
@@ -92,35 +92,41 @@ public abstract class IndexInput extends
/**
* Creates a random-access slice of this index input, with the given offset and length.
* <p>
- * The default implementation calls {@link #slice}, and implements absolute reads as
- * seek+read.
+ * The default implementation calls {@link #slice}, and it doesn't support random access,
+ * it implements absolute reads as seek+read.
*/
public RandomAccessInput randomAccessSlice(long offset, long length) throws IOException {
final IndexInput slice = slice("randomaccess", offset, length);
- return new RandomAccessInput() {
- @Override
- public byte readByte(long pos) throws IOException {
- slice.seek(pos);
- return slice.readByte();
- }
-
- @Override
- public short readShort(long pos) throws IOException {
- slice.seek(pos);
- return slice.readShort();
- }
-
- @Override
- public int readInt(long pos) throws IOException {
- slice.seek(pos);
- return slice.readInt();
- }
-
- @Override
- public long readLong(long pos) throws IOException {
- slice.seek(pos);
- return slice.readLong();
- }
- };
+ if (slice instanceof RandomAccessInput) {
+ // slice() already supports random access
+ return (RandomAccessInput) slice;
+ } else {
+ // return default impl
+ return new RandomAccessInput() {
+ @Override
+ public byte readByte(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readByte();
+ }
+
+ @Override
+ public short readShort(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readShort();
+ }
+
+ @Override
+ public int readInt(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readInt();
+ }
+
+ @Override
+ public long readLong(long pos) throws IOException {
+ slice.seek(pos);
+ return slice.readLong();
+ }
+ };
+ }
}
}