You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/11/07 19:43:51 UTC
svn commit: r1198865 - in /lucene/dev/trunk/lucene/src:
java/org/apache/lucene/index/values/ java/org/apache/lucene/util/packed/
test/org/apache/lucene/util/packed/
Author: mikemccand
Date: Mon Nov 7 18:43:50 2011
New Revision: 1198865
URL: http://svn.apache.org/viewvc?rev=1198865&view=rev
Log:
LUCENE-3524: add PackedInts.Reader impl that reads directly from Directory
Added:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/DirectReader.java (with props)
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java Mon Nov 7 18:43:50 2011
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
-import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -114,14 +113,14 @@ class FixedDerefBytesImpl {
}
final static class DirectFixedDerefSource extends DirectSource {
- private final PackedInts.RandomAccessReaderIterator index;
+ private final PackedInts.Reader index;
private final int size;
DirectFixedDerefSource(IndexInput data, IndexInput index, int size, ValueType type)
throws IOException {
super(data, type);
this.size = size;
- this.index = PackedInts.getRandomAccessReaderIterator(index);
+ this.index = PackedInts.getDirectReader(index);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java Mon Nov 7 18:43:50 2011
@@ -172,7 +172,7 @@ class FixedSortedBytesImpl {
}
static final class DirectFixedSortedSource extends SortedSource {
- final PackedInts.RandomAccessReaderIterator docToOrdIndex;
+ final PackedInts.Reader docToOrdIndex;
private final IndexInput datIn;
private final long basePointer;
private final int size;
@@ -182,7 +182,7 @@ class FixedSortedBytesImpl {
int valueCount, Comparator<BytesRef> comp, ValueType type)
throws IOException {
super(type, comp);
- docToOrdIndex = PackedInts.getRandomAccessReaderIterator(idxIn);
+ docToOrdIndex = PackedInts.getDirectReader(idxIn);
basePointer = datIn.getFilePointer();
this.datIn = datIn;
this.size = size;
@@ -191,11 +191,7 @@ class FixedSortedBytesImpl {
@Override
public int ord(int docID) {
- try {
- return (int) docToOrdIndex.get(docID);
- } catch (IOException e) {
- throw new IllegalStateException("failed to get ord", e);
- }
+ return (int) docToOrdIndex.get(docID);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java Mon Nov 7 18:43:50 2011
@@ -186,7 +186,7 @@ class PackedIntValues {
input = (IndexInput) datIn.clone();
if (values == null) {
- source = new PackedIntsSource(input);
+ source = new PackedIntsSource(input, false);
} else {
source = values.newFromInput(input, numDocs);
}
@@ -214,7 +214,7 @@ class PackedIntValues {
@Override
public Source getDirectSource() throws IOException {
- return values != null ? new FixedStraightBytesImpl.DirectFixedStraightSource((IndexInput) datIn.clone(), 8, ValueType.FIXED_INTS_64) : new DirectPackedIntsSource((IndexInput) datIn.clone());
+ return values != null ? new FixedStraightBytesImpl.DirectFixedStraightSource((IndexInput) datIn.clone(), 8, ValueType.FIXED_INTS_64) : new PackedIntsSource((IndexInput) datIn.clone(), true);
}
}
@@ -224,42 +224,13 @@ class PackedIntValues {
private final long defaultValue;
private final PackedInts.Reader values;
- public PackedIntsSource(IndexInput dataIn) throws IOException {
+ public PackedIntsSource(IndexInput dataIn, boolean direct) throws IOException {
super(ValueType.VAR_INTS);
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
- values = PackedInts.getReader(dataIn);
+ values = direct ? PackedInts.getDirectReader(dataIn) : PackedInts.getReader(dataIn);
}
-
- @Override
- public long getInt(int docID) {
- // TODO -- can we somehow avoid 2X method calls
- // on each get? must push minValue down, and make
- // PackedInts implement Ints.Source
- assert docID >= 0;
- final long value = values.get(docID);
- return value == defaultValue ? 0 : minValue + value;
- }
- }
-
- private static final class DirectPackedIntsSource extends Source {
- private final PackedInts.RandomAccessReaderIterator ints;
- private long minValue;
- private final long defaultValue;
-
- private DirectPackedIntsSource(IndexInput dataIn)
- throws IOException {
- super(ValueType.VAR_INTS);
- minValue = dataIn.readLong();
- defaultValue = dataIn.readLong();
- this.ints = PackedInts.getRandomAccessReaderIterator(dataIn);
- }
-
- @Override
- public double getFloat(int docID) {
- return getInt(docID);
- }
-
+
@Override
public BytesRef getBytes(int docID, BytesRef ref) {
ref.grow(8);
@@ -269,12 +240,12 @@ class PackedIntValues {
@Override
public long getInt(int docID) {
- try {
- final long val = ints.get(docID);
- return val == defaultValue ? 0 : minValue + val;
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ // TODO -- can we somehow avoid 2X method calls
+ // on each get? must push minValue down, and make
+ // PackedInts implement Ints.Source
+ assert docID >= 0;
+ final long value = values.get(docID);
+ return value == defaultValue ? 0 : minValue + value;
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Mon Nov 7 18:43:50 2011
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.DerefBytesWriterBase;
-import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -128,12 +127,12 @@ class VarDerefBytesImpl {
final static class DirectVarDerefSource extends DirectSource {
- private final PackedInts.RandomAccessReaderIterator index;
+ private final PackedInts.Reader index;
DirectVarDerefSource(IndexInput data, IndexInput index, ValueType type)
throws IOException {
super(data, type);
- this.index = PackedInts.getRandomAccessReaderIterator(index);
+ this.index = PackedInts.getDirectReader(index);
}
@Override
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Mon Nov 7 18:43:50 2011
@@ -133,6 +133,7 @@ final class VarSortedBytesImpl {
datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
offset += bytes.length;
}
+ // write sentinel
offsetWriter.add(offset);
offsetWriter.finish();
// write index
@@ -189,8 +190,8 @@ final class VarSortedBytesImpl {
}
private static final class DirectSortedSource extends SortedSource {
- private final PackedInts.RandomAccessReaderIterator docToOrdIndex;
- private final PackedInts.RandomAccessReaderIterator ordToOffsetIndex;
+ private final PackedInts.Reader docToOrdIndex;
+ private final PackedInts.Reader ordToOffsetIndex;
private final IndexInput datIn;
private final long basePointer;
private final int valueCount;
@@ -199,29 +200,26 @@ final class VarSortedBytesImpl {
Comparator<BytesRef> comparator, ValueType type) throws IOException {
super(type, comparator);
idxIn.readLong();
- ordToOffsetIndex = PackedInts.getRandomAccessReaderIterator(idxIn);
+ ordToOffsetIndex = PackedInts.getDirectReader(idxIn);
valueCount = ordToOffsetIndex.size()-1; // the last value here is just a dummy value to get the length of the last value
// advance this iterator to the end and clone the stream once it points to the docToOrdIndex header
- ordToOffsetIndex.advance(valueCount);
- docToOrdIndex = PackedInts.getRandomAccessReaderIterator((IndexInput) idxIn.clone()); // read the ords in to prevent too many random disk seeks
+ ordToOffsetIndex.get(valueCount);
+ docToOrdIndex = PackedInts.getDirectReader((IndexInput) idxIn.clone()); // read the ords in to prevent too many random disk seeks
basePointer = datIn.getFilePointer();
this.datIn = datIn;
}
@Override
public int ord(int docID) {
- try {
- return (int) docToOrdIndex.get(docID);
- } catch (IOException ex) {
- throw new IllegalStateException("failed", ex);
- }
+ return (int) docToOrdIndex.get(docID);
}
@Override
public BytesRef getByOrd(int ord, BytesRef bytesRef) {
try {
final long offset = ordToOffsetIndex.get(ord);
- final long nextOffset = ordToOffsetIndex.next();
+ // 1+ord is safe because we write a sentinel at the end
+ final long nextOffset = ordToOffsetIndex.get(1+ord);
datIn.seek(basePointer + offset);
final int length = (int) (nextOffset - offset);
bytesRef.grow(length);
@@ -231,7 +229,6 @@ final class VarSortedBytesImpl {
return bytesRef;
} catch (IOException ex) {
throw new IllegalStateException("failed", ex);
-
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Mon Nov 7 18:43:50 2011
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesSourceBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
-import org.apache.lucene.index.values.DirectSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -190,6 +189,7 @@ class VarStraightBytesImpl {
idxOut.writeVLong(0);
final PackedInts.Writer w = PackedInts.getWriter(idxOut, docCount+1,
PackedInts.bitsRequired(0));
+ // docCount+1 so we write sentinel
for (int i = 0; i < docCount+1; i++) {
w.add(0);
}
@@ -202,6 +202,7 @@ class VarStraightBytesImpl {
for (int i = 0; i < docCount; i++) {
w.add(docToAddress[i]);
}
+ // write sentinel
w.add(address);
w.finish();
}
@@ -262,20 +263,22 @@ class VarStraightBytesImpl {
public final static class DirectVarStraightSource extends DirectSource {
- private final PackedInts.RandomAccessReaderIterator index;
+ private final PackedInts.Reader index;
DirectVarStraightSource(IndexInput data, IndexInput index, ValueType type)
throws IOException {
super(data, type);
index.readVLong();
- this.index = PackedInts.getRandomAccessReaderIterator(index);
+ this.index = PackedInts.getDirectReader(index);
}
@Override
protected int position(int docID) throws IOException {
final long offset = index.get(docID);
data.seek(baseOffset + offset);
- return (int) (index.next() - offset);
+ // Safe to do 1+docID because we write sentinel at the end:
+ final long nextOffset = index.get(1+docID);
+ return (int) (nextOffset - offset);
}
}
}
Added: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/DirectReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/DirectReader.java?rev=1198865&view=auto
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/DirectReader.java (added)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/DirectReader.java Mon Nov 7 18:43:50 2011
@@ -0,0 +1,99 @@
+package org.apache.lucene.util.packed;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.IndexInput;
+
+import java.io.IOException;
+
+/* Reads directly from disk on each get */
+final class DirectReader implements PackedInts.Reader {
+ private final IndexInput in;
+ private final long startPointer;
+ private final int bitsPerValue;
+ private final int valueCount;
+
+ private static final int BLOCK_BITS = Packed64.BLOCK_BITS;
+ private static final int MOD_MASK = Packed64.MOD_MASK;
+
+ // masks[n-1] masks for bottom n bits
+ private final long[] masks;
+
+ public DirectReader(int bitsPerValue, int valueCount, IndexInput in)
+ throws IOException {
+ this.valueCount = valueCount;
+ this.bitsPerValue = bitsPerValue;
+ this.in = in;
+
+ long v = 1;
+ masks = new long[bitsPerValue];
+ for (int i = 0; i < bitsPerValue; i++) {
+ v *= 2;
+ masks[i] = v - 1;
+ }
+
+ startPointer = in.getFilePointer();
+ }
+
+ @Override
+ public int getBitsPerValue() {
+ return bitsPerValue;
+ }
+
+ @Override
+ public int size() {
+ return valueCount;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return false;
+ }
+
+ @Override
+ public Object getArray() {
+ return null;
+ }
+
+ @Override
+ public long get(int index) {
+ final long majorBitPos = (long)index * bitsPerValue;
+ final int elementPos = (int)(majorBitPos >>> BLOCK_BITS); // / BLOCK_SIZE
+ final int bitPos = (int)(majorBitPos & MOD_MASK); // % BLOCK_SIZE);
+
+ final long result;
+ try {
+ in.seek(startPointer + (elementPos << 3));
+ final long l1 = in.readLong();
+ final int bits1 = 64 - bitPos;
+ if (bits1 >= bitsPerValue) { // not split
+ result = l1 >> (bits1-bitsPerValue) & masks[bitsPerValue-1];
+ } else {
+ final int bits2 = bitsPerValue - bits1;
+ final long result1 = (l1 & masks[bits1-1]) << bits2;
+ final long l2 = in.readLong();
+ final long result2 = l2 >> (64 - bits2) & masks[bits2-1];
+ result = result1 | result2;
+ }
+
+ return result;
+ } catch (IOException ioe) {
+ throw new IllegalStateException("failed", ioe);
+ }
+ }
+}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/Packed64.java Mon Nov 7 18:43:50 2011
@@ -38,7 +38,7 @@ class Packed64 extends PackedInts.Reader
static final int MOD_MASK = BLOCK_SIZE - 1; // x % BLOCK_SIZE
private static final int ENTRY_SIZE = BLOCK_SIZE + 1;
- private static final int FAC_BITPOS = 3;
+ static final int FAC_BITPOS = 3;
/*
* In order to make an efficient value-getter, conditionals should be
@@ -50,10 +50,9 @@ class Packed64 extends PackedInts.Reader
* the right bits. By always shifting the second block right and applying
* a mask, we get the right bits there. After that, we | the two bitsets.
*/
- private static final int[][] SHIFTS =
- new int[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS];
- //new int[BLOCK_SIZE+1][BLOCK_SIZE][BLOCK_SIZE+1];
- private static final long[][] MASKS = new long[ENTRY_SIZE][ENTRY_SIZE];
+ static final int[][] SHIFTS =
+ new int[ENTRY_SIZE][ENTRY_SIZE * FAC_BITPOS];
+ static final long[][] MASKS = new long[ENTRY_SIZE][ENTRY_SIZE];
static { // Generate shifts
for (int elementBits = 1 ; elementBits <= BLOCK_SIZE ; elementBits++) {
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java Mon Nov 7 18:43:50 2011
@@ -104,14 +104,6 @@ public class PackedInts {
long advance(int ord) throws IOException;
}
- public static interface RandomAccessReaderIterator extends ReaderIterator {
- /**
- * @param index the position of the wanted value.
- * @return the value at the stated index.
- */
- long get(int index) throws IOException;
- }
-
/**
* A packed integer array that can be modified.
* @lucene.internal
@@ -230,22 +222,28 @@ public class PackedInts {
* @lucene.internal
*/
public static ReaderIterator getReaderIterator(IndexInput in) throws IOException {
- return getRandomAccessReaderIterator(in);
+ CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
+ final int bitsPerValue = in.readVInt();
+ assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
+ final int valueCount = in.readVInt();
+ return new PackedReaderIterator(bitsPerValue, valueCount, in);
}
/**
- * Retrieve PackedInts as a {@link RandomAccessReaderIterator}
+ * Retrieve PackedInts.Reader that does not load values
+ * into RAM but rather accesses all values via the
+ * provided IndexInput.
* @param in positioned at the beginning of a stored packed int structure.
- * @return an iterator to access the values
+ * @return an Reader to access the values
* @throws IOException if the structure could not be retrieved.
* @lucene.internal
*/
- public static RandomAccessReaderIterator getRandomAccessReaderIterator(IndexInput in) throws IOException {
+ public static Reader getDirectReader(IndexInput in) throws IOException {
CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
final int valueCount = in.readVInt();
- return new PackedReaderIterator(bitsPerValue, valueCount, in);
+ return new DirectReader(bitsPerValue, valueCount, in);
}
/**
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java Mon Nov 7 18:43:50 2011
@@ -21,15 +21,13 @@ import org.apache.lucene.store.IndexInpu
import java.io.IOException;
-final class PackedReaderIterator implements PackedInts.RandomAccessReaderIterator {
+final class PackedReaderIterator implements PackedInts.ReaderIterator {
private long pending;
private int pendingBitsLeft;
private final IndexInput in;
private final int bitsPerValue;
private final int valueCount;
private int position = -1;
- private long currentValue;
- private final long startPointer;
// masks[n-1] masks for bottom n bits
private final long[] masks;
@@ -41,7 +39,6 @@ final class PackedReaderIterator impleme
this.bitsPerValue = bitsPerValue;
this.in = in;
- startPointer = in.getFilePointer();
masks = new long[bitsPerValue];
long v = 1;
@@ -79,7 +76,7 @@ final class PackedReaderIterator impleme
}
++position;
- return currentValue = result;
+ return result;
}
public void close() throws IOException {
@@ -109,26 +106,6 @@ final class PackedReaderIterator impleme
pendingBitsLeft = 64 - (int)(skip % 64);
}
position = ord-1;
- return currentValue = next();
- }
-
-
- @Override
- public long get(int index) throws IOException {
- assert index < valueCount : "ord must be less than valueCount";
- if (index < position) {
- pendingBitsLeft = 0;
- final long bitsToSkip = (((long) bitsPerValue) * (long) index);
- final long skip = bitsToSkip - pendingBitsLeft;
- final long closestByte = (skip >> 6) << 3;
- in.seek(startPointer + closestByte);
- pending = in.readLong();
- pendingBitsLeft = 64 - (int) (skip % 64);
- position = index - 1;
- return currentValue = next();
- } else if (index == position) {
- return currentValue;
- }
- return advance(index);
+ return next();
}
}
Modified: lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1198865&r1=1198864&r2=1198865&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/trunk/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java Mon Nov 7 18:43:50 2011
@@ -116,29 +116,17 @@ public class TestPackedInts extends Luce
in.close();
}
- { // test reader iterator get
+ { // test direct reader get
IndexInput in = d.openInput("out.bin", newIOContext(random));
- PackedInts.RandomAccessReaderIterator intsEnum = PackedInts.getRandomAccessReaderIterator(in);
+ PackedInts.Reader intsEnum = PackedInts.getDirectReader(in);
for (int i = 0; i < valueCount; i++) {
final String msg = "index=" + i + " ceil=" + ceil + " valueCount="
+ valueCount + " nbits=" + nbits + " for "
+ intsEnum.getClass().getSimpleName();
- final int ord = random.nextInt(valueCount);
- long seek = intsEnum.get(ord);
- assertEquals(msg, seek, values[ord]);
- if (random.nextBoolean() && ord < valueCount-1) {
- if (random.nextBoolean()) {
- assertEquals(msg, values[ord+1], intsEnum.advance(ord+1));
- } else {
- assertEquals(msg, values[ord+1], intsEnum.next());
- }
- }
+ final int index = random.nextInt(valueCount);
+ long value = intsEnum.get(index);
+ assertEquals(msg, value, values[index]);
}
- if (intsEnum.ord() < valueCount - 1)
- assertEquals(values[valueCount - 1], intsEnum
- .advance(valueCount - 1));
- assertEquals(valueCount - 1, intsEnum.ord());
- assertEquals(fp, in.getFilePointer());
in.close();
}
ceil *= 2;