You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/07/07 16:18:33 UTC
svn commit: r1143840 [8/22] - in /lucene/dev/branches/flexscoring: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/
dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/contrib/i...
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java Thu Jul 7 14:18:02 2011
@@ -17,14 +17,20 @@ package org.apache.lucene.index.values;
* limitations under the License.
*/
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
+
import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.values.Bytes.BytesBaseSource;
import org.apache.lucene.index.values.Bytes.BytesReaderBase;
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.ByteBlockPool;
+import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
@@ -44,31 +50,60 @@ class FixedStraightBytesImpl {
private int size = -1;
// start at -1 if the first added value is > 0
private int lastDocID = -1;
- private byte[] oneRecord;
-
- public Writer(Directory dir, String id) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, false, null, null);
+ private final ByteBlockPool pool;
+ private boolean merge;
+ private final int byteBlockSize;
+ private IndexOutput datOut;
+
+ public Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException {
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
+ byteBlockSize = BYTE_BLOCK_SIZE;
}
-
@Override
public void add(int docID, BytesRef bytes) throws IOException {
+ assert lastDocID < docID;
+ assert !merge;
if (size == -1) {
+ if (bytes.length > BYTE_BLOCK_SIZE) {
+ throw new IllegalArgumentException("bytes arrays > " + Short.MAX_VALUE + " are not supported");
+ }
size = bytes.length;
- datOut.writeInt(size);
- oneRecord = new byte[size];
+ pool.nextBuffer();
} else if (bytes.length != size) {
throw new IllegalArgumentException("expected bytes size=" + size
+ " but got " + bytes.length);
}
- fill(docID);
- assert bytes.bytes.length >= bytes.length;
- datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ if (lastDocID+1 < docID) {
+ advancePool(docID);
+ }
+ pool.copy(bytes);
+ lastDocID = docID;
+ }
+
+ private final void advancePool(int docID) {
+ assert !merge;
+ long numBytes = (docID - (lastDocID+1))*size;
+ while(numBytes > 0) {
+ if (numBytes + pool.byteUpto < byteBlockSize) {
+ pool.byteUpto += numBytes;
+ numBytes = 0;
+ } else {
+ numBytes -= byteBlockSize - pool.byteUpto;
+ pool.nextBuffer();
+ }
+ }
+ assert numBytes == 0;
}
@Override
protected void merge(MergeState state) throws IOException {
- if (state.bits == null && state.reader instanceof Reader) {
+ merge = true;
+ datOut = getDataOut();
+ boolean success = false;
+ try {
+ if (state.liveDocs == null && state.reader instanceof Reader) {
Reader reader = (Reader) state.reader;
final int maxDocs = reader.maxDoc;
if (maxDocs == 0) {
@@ -77,48 +112,92 @@ class FixedStraightBytesImpl {
if (size == -1) {
size = reader.size;
datOut.writeInt(size);
- oneRecord = new byte[size];
}
- fill(state.docBase);
+ if (lastDocID+1 < state.docBase) {
+ fill(datOut, state.docBase);
+ lastDocID = state.docBase-1;
+ }
// TODO should we add a transfer to API to each reader?
final IndexInput cloneData = reader.cloneData();
try {
datOut.copyBytes(cloneData, size * maxDocs);
} finally {
- cloneData.close();
+ IOUtils.closeSafely(true, cloneData);
}
- lastDocID += maxDocs - 1;
+ lastDocID += maxDocs;
} else {
super.merge(state);
}
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeSafely(!success, datOut);
+ }
+ }
+ }
+
+
+
+ @Override
+ protected void mergeDoc(int docID) throws IOException {
+ assert lastDocID < docID;
+ if (size == -1) {
+ size = bytesRef.length;
+ datOut.writeInt(size);
+ }
+ assert size == bytesRef.length;
+ if (lastDocID+1 < docID) {
+ fill(datOut, docID);
+ }
+ datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+ lastDocID = docID;
}
+
+
// Fills up to but not including this docID
- private void fill(int docID) throws IOException {
+ private void fill(IndexOutput datOut, int docID) throws IOException {
assert size >= 0;
- for (int i = lastDocID + 1; i < docID; i++) {
- datOut.writeBytes(oneRecord, size);
+ final long numBytes = (docID - (lastDocID+1))*size;
+ final byte zero = 0;
+ for (long i = 0; i < numBytes; i++) {
+ datOut.writeByte(zero);
}
- lastDocID = docID;
}
@Override
public void finish(int docCount) throws IOException {
+ boolean success = false;
try {
- if (size == -1) {// no data added
- datOut.writeInt(0);
+ if (!merge) {
+ // indexing path - no disk IO until here
+ assert datOut == null;
+ datOut = getDataOut();
+ if (size == -1) {
+ datOut.writeInt(0);
+ } else {
+ datOut.writeInt(size);
+ pool.writePool(datOut);
+ }
+ if (lastDocID + 1 < docCount) {
+ fill(datOut, docCount);
+ }
} else {
- fill(docCount);
+ // merge path - datOut should be initialized
+ assert datOut != null;
+ if (size == -1) {// no data added
+ datOut.writeInt(0);
+ } else {
+ fill(datOut, docCount);
+ }
}
+ success = true;
} finally {
- super.finish(docCount);
+ pool.dropBuffersAndReset();
+ IOUtils.closeSafely(!success, datOut);
}
}
-
- public long ramBytesUsed() {
- return oneRecord == null ? 0 : oneRecord.length;
- }
}
public static class Reader extends BytesReaderBase {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Floats.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Floats.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Floats.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Floats.java Thu Jul 7 14:18:02 2011
@@ -25,10 +25,12 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.FloatsRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Exposes {@link Writer} and reader ({@link Source}) for 32 bit and 64 bit
@@ -44,11 +46,7 @@ public class Floats {
private static final String CODEC_NAME = "SimpleFloats";
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
- private static final int INT_DEFAULT = Float
- .floatToRawIntBits(0.0f);
- private static final long LONG_DEFAULT = Double
- .doubleToRawLongBits(0.0d);
-
+ private static final byte[] DEFAULTS = new byte[] {0,0,0,0,0,0,0,0};
public static Writer getWriter(Directory dir, String id, int precisionBytes,
AtomicLong bytesUsed) throws IOException {
@@ -70,16 +68,27 @@ public class Floats {
abstract static class FloatsWriter extends Writer {
private final String id;
- private FloatsRef floatsRef;
+ protected FloatsRef floatsRef;
protected int lastDocId = -1;
protected IndexOutput datOut;
private final byte precision;
+ private final Directory dir;
protected FloatsWriter(Directory dir, String id, int precision,
AtomicLong bytesUsed) throws IOException {
super(bytesUsed);
this.id = id;
this.precision = (byte) precision;
+ this.dir = dir;
+
+ }
+
+ public long ramBytesUsed() {
+ return 0;
+ }
+
+ final void initDataOut() throws IOException {
+ assert datOut == null;
datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
Writer.DATA_EXTENSION));
boolean success = false;
@@ -95,12 +104,8 @@ public class Floats {
}
}
- public long ramBytesUsed() {
- return 0;
- }
-
@Override
- protected void add(int docID) throws IOException {
+ protected void mergeDoc(int docID) throws IOException {
add(docID, floatsRef.get());
}
@@ -114,13 +119,20 @@ public class Floats {
floatsRef = valuesEnum.getFloat();
}
- protected abstract int fillDefault(int num) throws IOException;
+ protected final int fillDefault(int numValues) throws IOException {
+ for (int i = 0; i < numValues; i++) {
+ datOut.writeBytes(DEFAULTS, precision);
+ }
+ return numValues;
+ }
@Override
protected void merge(MergeState state) throws IOException {
- if (state.bits == null && state.reader instanceof FloatsReader) {
+ if (datOut == null) {
+ initDataOut();
+ }
+ if (state.liveDocs == null && state.reader instanceof FloatsReader) {
// no deletes - bulk copy
- // TODO: should be do bulks with deletes too?
final FloatsReader reader = (FloatsReader) state.reader;
assert reader.precisionBytes == (int) precision;
if (reader.maxDoc == 0)
@@ -131,8 +143,10 @@ public class Floats {
lastDocId += fillDefault(docBase - lastDocId - 1);
}
lastDocId += reader.transferTo(datOut);
- } else
- super.merge(state);
+ } else {
+ super.merge(state);
+ }
+
}
@Override
@@ -142,11 +156,13 @@ public class Floats {
}
// Writes 4 bytes (float) per value
- static class Float4Writer extends FloatsWriter {
-
+ static final class Float4Writer extends FloatsWriter {
+ private int[] values;
protected Float4Writer(Directory dir, String id, AtomicLong bytesUsed)
throws IOException {
super(dir, id, 4, bytesUsed);
+ values = new int[1];
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
}
@Override
@@ -154,76 +170,111 @@ public class Floats {
throws IOException {
assert docID > lastDocId : "docID: " + docID
+ " must be greater than the last added doc id: " + lastDocId;
+ if (docID >= values.length) {
+ final long len = values.length;
+ values = ArrayUtil.grow(values, 1 + docID);
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT
+ * ((values.length) - len));
+ }
+ values[docID] = Float.floatToRawIntBits((float)v);
+ lastDocId = docID;
+ }
+
+ @Override
+ protected void mergeDoc(int docID) throws IOException {
+ assert datOut != null;
+ assert docID > lastDocId : "docID: " + docID
+ + " must be greater than the last added doc id: " + lastDocId;
if (docID - lastDocId > 1) {
// fill with default values
- lastDocId += fillDefault(docID - lastDocId - 1);
+ fillDefault(docID - lastDocId - 1);
}
assert datOut != null;
- datOut.writeInt(Float.floatToRawIntBits((float) v));
- ++lastDocId;
+ datOut.writeInt(Float.floatToRawIntBits((float) floatsRef.get()));
+ lastDocId = docID;
}
@Override
public void finish(int docCount) throws IOException {
+ boolean success = false;
try {
- if (docCount > lastDocId + 1)
- for (int i = lastDocId; i < docCount; i++) {
- datOut.writeInt(INT_DEFAULT); // default value
+ int numDefaultsToAppend = docCount - (lastDocId + 1);
+ if (datOut == null) {
+ initDataOut();
+ for (int i = 0; i <= lastDocId; i++) {
+ datOut.writeInt(values[i]);
}
+ }
+ fillDefault(numDefaultsToAppend);
+ success = true;
} finally {
- datOut.close();
+ bytesUsed.addAndGet(-(RamUsageEstimator.NUM_BYTES_INT
+ * ((values.length))));
+ values = null;
+ IOUtils.closeSafely(!success, datOut);
}
}
- @Override
- protected int fillDefault(int numValues) throws IOException {
- for (int i = 0; i < numValues; i++) {
- datOut.writeInt(INT_DEFAULT);
- }
- return numValues;
- }
+
}
// Writes 8 bytes (double) per value
- static class Float8Writer extends FloatsWriter {
-
+ static final class Float8Writer extends FloatsWriter {
+ private long[] values;
protected Float8Writer(Directory dir, String id, AtomicLong bytesUsed)
throws IOException {
super(dir, id, 8, bytesUsed);
+ values = new long[1];
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG);
}
@Override
public void add(int docID, double v) throws IOException {
assert docID > lastDocId : "docID: " + docID
+ " must be greater than the last added doc id: " + lastDocId;
+ if (docID >= values.length) {
+ final long len = values.length;
+ values = ArrayUtil.grow(values, 1 + docID);
+ bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG
+ * ((values.length) - len));
+ }
+ values[docID] = Double.doubleToLongBits(v);
+ lastDocId = docID;
+ }
+
+ @Override
+ protected void mergeDoc(int docID) throws IOException {
+ assert docID > lastDocId : "docID: " + docID
+ + " must be greater than the last added doc id: " + lastDocId;
if (docID - lastDocId > 1) {
// fill with default values
lastDocId += fillDefault(docID - lastDocId - 1);
}
assert datOut != null;
- datOut.writeLong(Double.doubleToRawLongBits(v));
- ++lastDocId;
+ datOut.writeLong(Double.doubleToRawLongBits((float) floatsRef.get()));
+ lastDocId = docID;
}
@Override
public void finish(int docCount) throws IOException {
+ boolean success = false;
try {
- if (docCount > lastDocId + 1)
- for (int i = lastDocId; i < docCount; i++) {
- datOut.writeLong(LONG_DEFAULT); // default value
+ int numDefaultsToAppend = docCount - (lastDocId + 1);
+ if (datOut == null) {
+ initDataOut();
+ for (int i = 0; i <= lastDocId; i++) {
+ datOut.writeLong(values[i]);
}
+ }
+ fillDefault(numDefaultsToAppend);
+ success = true;
} finally {
- datOut.close();
+ bytesUsed.addAndGet(-(RamUsageEstimator.NUM_BYTES_LONG
+ * ((values.length))));
+ values = null;
+ IOUtils.closeSafely(!success, datOut);
}
}
-
- @Override
- protected int fillDefault(int numValues) throws IOException {
- for (int i = 0; i < numValues; i++) {
- datOut.writeLong(LONG_DEFAULT);
- }
- return numValues;
- }
}
/**
@@ -290,7 +341,7 @@ public class Floats {
}
}
- private class Source4 extends Source {
+ private final class Source4 extends Source {
private final float[] values;
Source4(final float[] values ) throws IOException {
@@ -317,12 +368,22 @@ public class Floats {
}
@Override
+ public Object getArray() {
+ return this.values;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return true;
+ }
+
+ @Override
public ValueType type() {
return ValueType.FLOAT_32;
}
}
- private class Source8 extends Source {
+ private final class Source8 extends Source {
private final double[] values;
Source8(final double[] values) throws IOException {
@@ -352,6 +413,16 @@ public class Floats {
public ValueType type() {
return ValueType.FLOAT_64;
}
+
+ @Override
+ public Object getArray() {
+ return this.values;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return true;
+ }
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java Thu Jul 7 14:18:02 2011
@@ -254,6 +254,25 @@ public abstract class IndexDocValues imp
*/
public abstract ValuesEnum getEnum(AttributeSource attrSource)
throws IOException;
+
+ /**
+ * Returns <code>true</code> iff this {@link Source} exposes an array via
+ * {@link #getArray()} otherwise <code>false</code>.
+ *
+ * @return <code>true</code> iff this {@link Source} exposes an array via
+ * {@link #getArray()} otherwise <code>false</code>.
+ */
+ public boolean hasArray() {
+ return false;
+ }
+
+ /**
+ * Returns the internal array representation iff this {@link Source} uses an
+ * array as its inner representation, otherwise <code>null</code>.
+ */
+ public Object getArray() {
+ return null;
+ }
}
/**
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Ints.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Ints.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Ints.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Ints.java Thu Jul 7 14:18:02 2011
@@ -33,14 +33,11 @@ public class Ints {
private Ints() {
}
- public static Writer getWriter(Directory dir, String id,
- boolean useFixedArray, AtomicLong bytesUsed) throws IOException {
- // TODO - implement fixed?!
- return new IntsWriter(dir, id, bytesUsed);
+ public static Writer getWriter(Directory dir, String id, AtomicLong bytesUsed, ValueType type) throws IOException {
+ return new IntsWriter(dir, id, bytesUsed, type);
}
- public static IndexDocValues getValues(Directory dir, String id,
- boolean useFixedArray) throws IOException {
- return new IntsReader(dir, id);
+ public static IndexDocValues getValues(Directory dir, String id, int numDocs) throws IOException {
+ return new IntsReader(dir, id, numDocs);
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java Thu Jul 7 14:18:02 2011
@@ -21,19 +21,24 @@ import java.util.Collection;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.values.IndexDocValuesArray;
+import org.apache.lucene.index.values.IndexDocValues.Source;
+import org.apache.lucene.index.values.IndexDocValues.SourceEnum;
+import org.apache.lucene.index.values.IndexDocValuesArray.ByteValues;
+import org.apache.lucene.index.values.IndexDocValuesArray.IntValues;
+import org.apache.lucene.index.values.IndexDocValuesArray.LongValues;
+import org.apache.lucene.index.values.IndexDocValuesArray.ShortValues;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LongsRef;
-import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.packed.PackedInts;
/**
- * Stores ints packed with fixed-bit precision.
+ * Stores ints packed and fixed with fixed-bit precision.
*
* @lucene.experimental
* */
@@ -41,45 +46,60 @@ class IntsImpl {
private static final String CODEC_NAME = "Ints";
private static final byte PACKED = 0x00;
- private static final byte FIXED = 0x01;
-
+ private static final byte FIXED_64 = 0x01;
+ private static final byte FIXED_32 = 0x02;
+ private static final byte FIXED_16 = 0x03;
+ private static final byte FIXED_8 = 0x04;
+
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
static class IntsWriter extends Writer {
- // TODO: can we bulkcopy this on a merge?
private LongsRef intsRef;
- private long[] docToValue;
+ private final IndexDocValuesArray array;
private long minValue;
private long maxValue;
private boolean started;
private final String id;
private int lastDocId = -1;
+ private final Directory dir;
+ private final byte typeOrd;
private IndexOutput datOut;
+ private boolean merging;
+
- protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed)
- throws IOException {
+ protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed,
+ ValueType valueType) throws IOException {
super(bytesUsed);
- datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
- DATA_EXTENSION));
- boolean success = false;
- try {
- CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
- this.id = id;
- docToValue = new long[1];
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG); // TODO the
- // bitset
- // needs memory
- // too
- success = true;
- } finally {
- if (!success) {
- datOut.close();
- }
+ this.dir = dir;
+ this.id = id;
+ switch (valueType) {
+ case FIXED_INTS_16:
+ array= new ShortValues(bytesUsed);
+ typeOrd = FIXED_16;
+ break;
+ case FIXED_INTS_32:
+ array = new IntValues(bytesUsed);
+ typeOrd = FIXED_32;
+ break;
+ case FIXED_INTS_64:
+ array = new LongValues(bytesUsed);
+ typeOrd = FIXED_64;
+ break;
+ case FIXED_INTS_8:
+ array = new ByteValues(bytesUsed);
+ typeOrd = FIXED_8;
+ break;
+ case VAR_INTS:
+ array = new LongValues(bytesUsed);
+ typeOrd = PACKED;
+ break;
+ default:
+ throw new IllegalStateException("unknown type " + valueType);
}
}
-
+
@Override
public void add(int docID, long v) throws IOException {
assert lastDocId < docID;
@@ -94,71 +114,138 @@ class IntsImpl {
}
}
lastDocId = docID;
-
- if (docID >= docToValue.length) {
- final long len = docToValue.length;
- docToValue = ArrayUtil.grow(docToValue, 1 + docID);
- bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_LONG
- * ((docToValue.length) - len));
+ array.set(docID, v);
+ }
+
+ private final void initDataOut(byte typeOrd) throws IOException {
+ if (datOut == null) {
+ boolean success = false;
+ try {
+ datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "",
+ DATA_EXTENSION));
+ CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT);
+ datOut.writeByte(typeOrd);
+ success = true;
+
+ } finally {
+ if (!success) {
+ IOUtils.closeSafely(true, datOut);
+ }
+ }
}
- docToValue[docID] = v;
}
@Override
public void finish(int docCount) throws IOException {
+ boolean success = false;
try {
- if (!started) {
- minValue = maxValue = 0;
- }
- // if we exceed the range of positive longs we must switch to fixed ints
- if ((maxValue - minValue) < (((long)1) << 63) && (maxValue - minValue) >= 0) {
- writePackedInts(docCount);
+ if (datOut == null) {
+ // if we only add or merge Packed ints datOut is not initialized
+ assert !merging || typeOrd == PACKED;
+ finishAdd(docCount);
} else {
- writeFixedInts(docCount);
+ assert datOut != null && merging && typeOrd != PACKED;
+ // on merge, simply fill up missing values
+ fillDefault(datOut, docCount - (lastDocId + 1));
}
-
+ success = true;
} finally {
- datOut.close();
- bytesUsed
- .addAndGet(-(RamUsageEstimator.NUM_BYTES_LONG * docToValue.length));
- docToValue = null;
+ IOUtils.closeSafely(!success, datOut);
+ array.clear();
}
}
-
- private void writeFixedInts(int docCount) throws IOException {
- datOut.writeByte(FIXED);
- datOut.writeInt(docCount);
- for (int i = 0; i < docToValue.length; i++) {
- datOut.writeLong(docToValue[i]); // write full array - we use 0 as default
+
+ private final void finishAdd(int docCount) throws IOException {
+ if (!started) {
+ minValue = maxValue = 0;
}
- for (int i = docToValue.length; i < docCount; i++) {
- datOut.writeLong(0); // fill with defaults values
+ byte headerType = typeOrd;
+ if (typeOrd == PACKED) {
+ final long delta = maxValue - minValue;
+ // if we exceed the range of positive longs we must switch to fixed
+ // ints
+ if (delta <= (maxValue >= 0 && minValue <= 0 ? Long.MAX_VALUE
+ : Long.MAX_VALUE - 1) && delta >= 0) {
+ writePackedInts(docCount);
+ return; // done
+ } else {
+ headerType = FIXED_64;
+ }
+ }
+ initDataOut(headerType);
+ array.write(datOut, docCount);
+ assert datOut != null;
+ }
+ // TODO how can we improve VAR_INT mergeing here without violating compression?
+ @Override
+ protected void merge(MergeState state) throws IOException {
+ merging = true;
+ if (typeOrd != PACKED) {
+ initDataOut(typeOrd); // init datOut since we merge directly
+ if (state.liveDocs == null && state.reader instanceof IntsReader) {
+ // no deleted docs - try bulk copy
+ final IntsReader reader = (IntsReader) state.reader;
+ if (reader.type == typeOrd) {
+ final int docBase = state.docBase;
+ if (docBase - lastDocId > 1) {
+ // fill with default values
+ lastDocId += fillDefault(datOut, docBase - lastDocId - 1);
+ }
+ lastDocId += reader.transferTo(datOut);
+ return;
+ }
+ }
}
+ super.merge(state);
+ }
+
+ @Override
+ protected void mergeDoc(int docID) throws IOException {
+ assert docID > lastDocId : "docID: " + docID
+ + " must be greater than the last added doc id: " + lastDocId;
+ assert merging;
+ final long value = intsRef.get();
+ if (typeOrd != PACKED) {
+ // if now packed we do straight merging and write values directly
+ assert datOut != null;
+ if (docID - lastDocId > 1) {
+ // fill with default values
+ array.writeDefaults(datOut, docID - lastDocId - 1);
+ }
+ array.writeDirect(datOut, value);
+ lastDocId = docID;
+ } else {
+ add(docID, value);
+ }
+ }
+
+ protected final int fillDefault(IndexOutput datOut, int numValues) throws IOException {
+ array.writeDefaults(datOut, numValues);
+ return numValues;
}
private void writePackedInts(int docCount) throws IOException {
- datOut.writeByte(PACKED);
+ initDataOut(PACKED);
datOut.writeLong(minValue);
+ assert array.type() == ValueType.FIXED_INTS_64;
+ final long[] docToValue = (long[])array.getArray();
// write a default value to recognize docs without a value for that
// field
- final long defaultValue = maxValue>= 0 && minValue <=0 ? 0-minValue : ++maxValue-minValue;
+ final long defaultValue = maxValue >= 0 && minValue <= 0 ? 0 - minValue
+ : ++maxValue - minValue;
datOut.writeLong(defaultValue);
PackedInts.Writer w = PackedInts.getWriter(datOut, docCount,
- PackedInts.bitsRequired(maxValue-minValue));
- final int limit = docToValue.length > docCount ? docCount : docToValue.length;
+ PackedInts.bitsRequired(maxValue - minValue));
+ final int limit = docToValue.length > docCount ? docCount
+ : docToValue.length;
for (int i = 0; i < limit; i++) {
w.add(docToValue[i] == 0 ? defaultValue : docToValue[i] - minValue);
}
for (int i = limit; i < docCount; i++) {
w.add(defaultValue);
}
-
- w.finish();
- }
- @Override
- protected void add(int docID) throws IOException {
- add(docID, intsRef.get());
+ w.finish();
}
@Override
@@ -183,15 +270,17 @@ class IntsImpl {
*/
static class IntsReader extends IndexDocValues {
private final IndexInput datIn;
- private final boolean packed;
+ private final byte type;
+ private final int numDocs;
- protected IntsReader(Directory dir, String id) throws IOException {
+ protected IntsReader(Directory dir, String id, int numDocs) throws IOException {
datIn = dir.openInput(IndexFileNames.segmentFileName(id, "",
Writer.DATA_EXTENSION));
+ this.numDocs = numDocs;
boolean success = false;
try {
CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START);
- packed = PACKED == datIn.readByte();
+ type = datIn.readByte();
success = true;
} finally {
if (!success) {
@@ -200,105 +289,41 @@ class IntsImpl {
}
}
+ public int transferTo(IndexOutput datOut) throws IOException {
+ IndexInput indexInput = (IndexInput) datIn.clone();
+ boolean success = false;
+ try {
+ indexInput.seek(CodecUtil.headerLength(CODEC_NAME));
+ // skip type
+ indexInput.readByte();
+ datOut.copyBytes(indexInput, bytesPerValue(type) * numDocs);
+ success = true;
+ } finally {
+ IOUtils.closeSafely(!success, indexInput);
+ }
+ return numDocs;
+ }
+
/**
* Loads the actual values. You may call this more than once, eg if you
* already previously loaded but then discarded the Source.
*/
@Override
public Source load() throws IOException {
- final IndexInput input = (IndexInput) datIn.clone();
boolean success = false;
+ final Source source;
+ IndexInput input = null;
try {
- final Source source = packed ? new PackedIntsSource(input)
- : new FixedIntsSource(input);
+ input = (IndexInput) datIn.clone();
+ input.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+ source = loadFixedSource(type, input, numDocs);
success = true;
return source;
} finally {
if (!success) {
- IOUtils.closeSafely(true, datIn);
- }
- }
- }
-
- private static class FixedIntsSource extends Source {
- private final long[] values;
- public FixedIntsSource(IndexInput dataIn) throws IOException {
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
- final int numDocs = dataIn.readInt();
- values = new long[numDocs];
- for (int i = 0; i < values.length; i++) {
- values[i] = dataIn.readLong();
+ IOUtils.closeSafely(true, input, datIn);
}
}
-
- @Override
- public long getInt(int docID) {
- assert docID >= 0 && docID < values.length;
- return values[docID];
- }
-
- @Override
- public ValueType type() {
- return ValueType.INTS;
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource)
- throws IOException {
- return new SourceEnum(attrSource, type(), this, values.length) {
-
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- intsRef.ints[intsRef.offset] = values[target];
- return pos = target;
- }
- };
- }
-
- }
-
- private static class PackedIntsSource extends Source {
- private final long minValue;
- private final long defaultValue;
- private final PackedInts.Reader values;
-
- public PackedIntsSource(IndexInput dataIn) throws IOException {
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
- minValue = dataIn.readLong();
- defaultValue = dataIn.readLong();
- values = PackedInts.getReader(dataIn);
- }
-
- @Override
- public long getInt(int docID) {
- // TODO -- can we somehow avoid 2X method calls
- // on each get? must push minValue down, and make
- // PackedInts implement Ints.Source
- assert docID >= 0;
- final long value = values.get(docID);
- return value == defaultValue ? 0 : minValue + value;
- }
-
- @Override
- public ValuesEnum getEnum(AttributeSource attrSource)
- throws IOException {
- return new SourceEnum(attrSource, type(), this, values.size()) {
- @Override
- public int advance(int target) throws IOException {
- if (target >= numDocs)
- return pos = NO_MORE_DOCS;
- intsRef.ints[intsRef.offset] = source.getInt(target);
- return pos = target;
- }
- };
- }
-
- @Override
- public ValueType type() {
- return ValueType.INTS;
- }
}
@Override
@@ -312,8 +337,8 @@ class IntsImpl {
final IndexInput input = (IndexInput) datIn.clone();
boolean success = false;
try {
- ValuesEnum inst = packed ? new PackedIntsEnumImpl(source, input)
- : new FixedIntsEnumImpl(source, input);
+ input.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+ final ValuesEnum inst = directEnum(type, source, input, numDocs);
success = true;
return inst;
} finally {
@@ -325,77 +350,129 @@ class IntsImpl {
@Override
public ValueType type() {
- return ValueType.INTS;
+ return ValueType.VAR_INTS;
+ }
+ }
+
+ private static ValuesEnum directEnum(byte ord, AttributeSource attrSource, IndexInput input, int numDocs) throws IOException {
+ switch (ord) {
+ case FIXED_16:
+ return new ShortValues((AtomicLong)null).getDirectEnum(attrSource, input, numDocs);
+ case FIXED_32:
+ return new IntValues((AtomicLong)null).getDirectEnum(attrSource, input, numDocs);
+ case FIXED_64:
+ return new LongValues((AtomicLong)null).getDirectEnum(attrSource, input, numDocs);
+ case FIXED_8:
+ return new ByteValues((AtomicLong)null).getDirectEnum(attrSource, input, numDocs);
+ case PACKED:
+ return new PackedIntsEnumImpl(attrSource, input);
+ default:
+ throw new IllegalStateException("unknown type ordinal " + ord);
+ }
+ }
+
+ private static IndexDocValues.Source loadFixedSource(byte ord, IndexInput input, int numDoc) throws IOException {
+ switch (ord) {
+ case FIXED_16:
+ return new ShortValues(input, numDoc);
+ case FIXED_32:
+ return new IntValues(input, numDoc);
+ case FIXED_64:
+ return new LongValues(input, numDoc);
+ case FIXED_8:
+ return new ByteValues(input, numDoc);
+ case PACKED:
+ return new PackedIntsSource(input);
+ default:
+ throw new IllegalStateException("unknown type ordinal " + ord);
}
-
+ }
+
+ private static int bytesPerValue(byte typeOrd) {
+ final int numBytes;
+ switch (typeOrd) {
+ case FIXED_16:
+ numBytes = 2;
+ break;
+ case FIXED_32:
+ numBytes = 4;
+ break;
+ case FIXED_64:
+ numBytes = 8;
+ break;
+ case FIXED_8:
+ numBytes = 1;
+ break;
+ default:
+ throw new IllegalStateException("illegal type ord " + typeOrd);
+ }
+ return numBytes;
}
- private static final class PackedIntsEnumImpl extends ValuesEnum {
- private final PackedInts.ReaderIterator ints;
- private long minValue;
- private final IndexInput dataIn;
+ static class PackedIntsSource extends Source {
+ private final long minValue;
private final long defaultValue;
- private final int maxDoc;
- private int pos = -1;
+ private final PackedInts.Reader values;
- private PackedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
- throws IOException {
- super(source, ValueType.INTS);
- intsRef.offset = 0;
- this.dataIn = dataIn;
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
+ public PackedIntsSource(IndexInput dataIn) throws IOException {
+
minValue = dataIn.readLong();
defaultValue = dataIn.readLong();
- this.ints = PackedInts.getReaderIterator(dataIn);
- maxDoc = ints.size();
+ values = PackedInts.getReader(dataIn);
}
@Override
- public void close() throws IOException {
- ints.close();
- dataIn.close();
+ public long getInt(int docID) {
+ // TODO -- can we somehow avoid 2X method calls
+ // on each get? must push minValue down, and make
+ // PackedInts implement Ints.Source
+ assert docID >= 0;
+ final long value = values.get(docID);
+ return value == defaultValue ? 0 : minValue + value;
}
@Override
- public int advance(int target) throws IOException {
- if (target >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- final long val = ints.advance(target);
- intsRef.ints[intsRef.offset] = val == defaultValue ? 0 : minValue + val;
- return pos = target;
- }
-
- @Override
- public int docID() {
- return pos;
+ public ValuesEnum getEnum(AttributeSource attrSource) throws IOException {
+ return new SourceEnum(attrSource, type(), this, values.size()) {
+ @Override
+ public int advance(int target) throws IOException {
+ if (target >= numDocs)
+ return pos = NO_MORE_DOCS;
+ intsRef.ints[intsRef.offset] = source.getInt(target);
+ return pos = target;
+ }
+ };
}
@Override
- public int nextDoc() throws IOException {
- if (pos >= maxDoc) {
- return pos = NO_MORE_DOCS;
- }
- return advance(pos + 1);
+ public ValueType type() {
+ return ValueType.VAR_INTS;
}
}
-
- private static final class FixedIntsEnumImpl extends ValuesEnum {
+
+
+ private static final class PackedIntsEnumImpl extends ValuesEnum {
+ private final PackedInts.ReaderIterator ints;
+ private long minValue;
private final IndexInput dataIn;
+ private final long defaultValue;
private final int maxDoc;
private int pos = -1;
- private FixedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
+ private PackedIntsEnumImpl(AttributeSource source, IndexInput dataIn)
throws IOException {
- super(source, ValueType.INTS);
+ super(source, ValueType.VAR_INTS);
intsRef.offset = 0;
this.dataIn = dataIn;
- dataIn.seek(CodecUtil.headerLength(CODEC_NAME) + 1);
- maxDoc = dataIn.readInt();
+ minValue = dataIn.readLong();
+ defaultValue = dataIn.readLong();
+ this.ints = PackedInts.getReaderIterator(dataIn);
+ maxDoc = ints.size();
}
@Override
public void close() throws IOException {
+ ints.close();
dataIn.close();
}
@@ -404,11 +481,8 @@ class IntsImpl {
if (target >= maxDoc) {
return pos = NO_MORE_DOCS;
}
- assert target > pos;
- if (target > pos+1) {
- dataIn.seek(dataIn.getFilePointer() + ((target - pos - 1) * 8));
- }
- intsRef.ints[intsRef.offset] = dataIn.readLong();
+ final long val = ints.advance(target);
+ intsRef.ints[intsRef.offset] = val == defaultValue ? 0 : minValue + val;
return pos = target;
}
@@ -425,5 +499,7 @@ class IntsImpl {
return advance(pos + 1);
}
}
+
+
}
\ No newline at end of file
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValueType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValueType.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValueType.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValueType.java Thu Jul 7 14:18:02 2011
@@ -18,8 +18,8 @@ package org.apache.lucene.index.values;
*/
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.values.IndexDocValues.SortedSource;
+import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.packed.PackedInts;
@@ -32,16 +32,40 @@ import org.apache.lucene.util.packed.Pac
* @lucene.experimental
*/
public enum ValueType {
- /*
- * TODO: Add INT_32 INT_64 INT_16 & INT_8?!
- */
+
/**
- * A 64 bit integer value. By default this type uses
+ * A variable bit signed integer value. By default this type uses
* {@link PackedInts} to compress the values, as an offset
* from the minimum value, as long as the value range
* fits into 2<sup>63</sup>-1. Otherwise,
* the default implementation falls back to fixed size 64bit
- * integers.
+ * integers ({@link #FIXED_INTS_64}).
+ * <p>
+ * NOTE: this type uses <tt>0</tt> as the default value without any
+ * distinction between provided <tt>0</tt> values during indexing. All
+ * documents without an explicit value will use <tt>0</tt> instead. In turn,
+ * {@link ValuesEnum} instances will not skip documents without an explicit
+ * value assigned. Custom default values must be assigned explicitly.
+ * </p>
+ */
+ VAR_INTS,
+
+ /**
+ * A 8 bit signed integer value. {@link Source} instances of
+ * this type return a <tt>byte</tt> array from {@link Source#getArray()}
+ * <p>
+ * NOTE: this type uses <tt>0</tt> as the default value without any
+ * distinction between provided <tt>0</tt> values during indexing. All
+ * documents without an explicit value will use <tt>0</tt> instead. In turn,
+ * {@link ValuesEnum} instances will not skip documents without an explicit
+ * value assigned. Custom default values must be assigned explicitly.
+ * </p>
+ */
+ FIXED_INTS_8,
+
+ /**
+ * A 16 bit signed integer value. {@link Source} instances of
+ * this type return a <tt>short</tt> array from {@link Source#getArray()}
* <p>
* NOTE: this type uses <tt>0</tt> as the default value without any
* distinction between provided <tt>0</tt> values during indexing. All
@@ -50,13 +74,39 @@ public enum ValueType {
* value assigned. Custom default values must be assigned explicitly.
* </p>
*/
- INTS,
+ FIXED_INTS_16,
+
+ /**
+ * A 32 bit signed integer value. {@link Source} instances of
+ * this type return a <tt>int</tt> array from {@link Source#getArray()}
+ * <p>
+ * NOTE: this type uses <tt>0</tt> as the default value without any
+ * distinction between provided <tt>0</tt> values during indexing. All
+ * documents without an explicit value will use <tt>0</tt> instead. In turn,
+ * {@link ValuesEnum} instances will not skip documents without an explicit
+ * value assigned. Custom default values must be assigned explicitly.
+ * </p>
+ */
+ FIXED_INTS_32,
/**
+ * A 64 bit signed integer value. {@link Source} instances of
+ * this type return a <tt>long</tt> array from {@link Source#getArray()}
+ * <p>
+ * NOTE: this type uses <tt>0</tt> as the default value without any
+ * distinction between provided <tt>0</tt> values during indexing. All
+ * documents without an explicit value will use <tt>0</tt> instead. In turn,
+ * {@link ValuesEnum} instances will not skip documents without an explicit
+ * value assigned. Custom default values must be assigned explicitly.
+ * </p>
+ */
+ FIXED_INTS_64,
+ /**
* A 32 bit floating point value. By default there is no compression
* applied. To fit custom float values into less than 32bit either a custom
* implementation is needed or values must be encoded into a
- * {@link #BYTES_FIXED_STRAIGHT} type.
+ * {@link #BYTES_FIXED_STRAIGHT} type. {@link Source} instances of
+ * this type return a <tt>float</tt> array from {@link Source#getArray()}
* <p>
* NOTE: this type uses <tt>0.0f</tt> as the default value without any
* distinction between provided <tt>0.0f</tt> values during indexing. All
@@ -67,10 +117,12 @@ public enum ValueType {
*/
FLOAT_32,
/**
+ *
* A 64 bit floating point value. By default there is no compression
* applied. To fit custom float values into less than 64bit either a custom
* implementation is needed or values must be encoded into a
- * {@link #BYTES_FIXED_STRAIGHT} type.
+ * {@link #BYTES_FIXED_STRAIGHT} type. {@link Source} instances of
+ * this type return a <tt>double</tt> array from {@link Source#getArray()}
* <p>
* NOTE: this type uses <tt>0.0d</tt> as the default value without any
* distinction between provided <tt>0.0d</tt> values during indexing. All
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/ValuesEnum.java Thu Jul 7 14:18:02 2011
@@ -71,13 +71,18 @@ public abstract class ValuesEnum extends
case BYTES_VAR_STRAIGHT:
bytesRef = new BytesRef();
break;
- case INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ case VAR_INTS:
intsRef = new LongsRef(1);
break;
case FLOAT_32:
case FLOAT_64:
floatsRef = new FloatsRef(1);
break;
+
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java Thu Jul 7 14:18:02 2011
@@ -27,12 +27,14 @@ import org.apache.lucene.index.values.Fi
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.CodecUtil;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.ByteBlockPool.Allocator;
@@ -113,7 +115,7 @@ class VarDerefBytesImpl {
private final AddressByteStartArray array = new AddressByteStartArray(1,
bytesUsed);
- private final BytesRefHash hash = new BytesRefHash(pool, 16, array);
+ private final BytesRefHash hash;
public Writer(Directory dir, String id, AtomicLong bytesUsed)
throws IOException {
@@ -123,8 +125,8 @@ class VarDerefBytesImpl {
public Writer(Directory dir, String id, Allocator allocator,
AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, true,
- new ByteBlockPool(allocator), bytesUsed);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ hash = new BytesRefHash(new ByteBlockPool(allocator), 16, array);
docToAddress = new int[1];
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
}
@@ -144,8 +146,7 @@ class VarDerefBytesImpl {
final int docAddress;
if (e >= 0) {
docAddress = array.address[e] = address;
- address += writePrefixLength(datOut, bytes);
- datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ address += bytes.length < 128 ? 1 : 2;
address += bytes.length;
} else {
docAddress = array.address[(-e) - 1];
@@ -169,6 +170,24 @@ class VarDerefBytesImpl {
// some last docs that we didn't see
@Override
public void finish(int docCount) throws IOException {
+ final IndexOutput datOut = getDataOut();
+ boolean success = false;
+ try {
+ final int size = hash.size();
+ final BytesRef bytesRef = new BytesRef();
+ for (int i = 0; i < size; i++) {
+ hash.get(i, bytesRef);
+ writePrefixLength(datOut, bytesRef);
+ datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+ }
+ success = true;
+ } finally {
+ hash.close();
+ IOUtils.closeSafely(!success, datOut);
+ }
+
+ final IndexOutput idxOut = getIndexOut();
+ success = false;
try {
idxOut.writeInt(address - 1);
// write index
@@ -189,9 +208,9 @@ class VarDerefBytesImpl {
w.add(0);
}
w.finish();
+ success = true;
} finally {
- hash.close();
- super.finish(docCount);
+ IOUtils.closeSafely(!success,idxOut);
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT
* (-docToAddress.length));
docToAddress = null;
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Thu Jul 7 14:18:02 2011
@@ -27,11 +27,13 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.ByteBlockPool.Allocator;
@@ -56,9 +58,7 @@ class VarSortedBytesImpl {
private int[] docToEntry;
private final Comparator<BytesRef> comp;
- private final BytesRefHash hash = new BytesRefHash(pool,
- BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
- BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
+ private final BytesRefHash hash;
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
AtomicLong bytesUsed) throws IOException {
@@ -68,13 +68,14 @@ class VarSortedBytesImpl {
public Writer(Directory dir, String id, Comparator<BytesRef> comp,
Allocator allocator, AtomicLong bytesUsed) throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, true,
- new ByteBlockPool(allocator), bytesUsed);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ this.hash = new BytesRefHash(new ByteBlockPool(allocator),
+ BytesRefHash.DEFAULT_CAPACITY, new TrackingDirectBytesStartArray(
+ BytesRefHash.DEFAULT_CAPACITY, bytesUsed));
this.comp = comp;
docToEntry = new int[1];
docToEntry[0] = -1;
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
-
}
@Override
@@ -99,14 +100,16 @@ class VarSortedBytesImpl {
@Override
public void finish(int docCount) throws IOException {
final int count = hash.size();
+ final IndexOutput datOut = getDataOut();
+ long offset = 0;
+ long lastOffset = 0;
+ final int[] index = new int[count];
+ final long[] offsets = new long[count];
+ boolean success = false;
try {
final int[] sortedEntries = hash.sort(comp);
// first dump bytes data, recording index & offset as
// we go
- long offset = 0;
- long lastOffset = 0;
- final int[] index = new int[count];
- final long[] offsets = new long[count];
for (int i = 0; i < count; i++) {
final int e = sortedEntries[i];
offsets[i] = offset;
@@ -118,7 +121,14 @@ class VarSortedBytesImpl {
lastOffset = offset;
offset += bytes.length;
}
-
+ success = true;
+ } finally {
+ IOUtils.closeSafely(!success, datOut);
+ hash.close();
+ }
+ final IndexOutput idxOut = getIndexOut();
+ success = false;
+ try {
// total bytes of data
idxOut.writeLong(offset);
@@ -145,11 +155,12 @@ class VarSortedBytesImpl {
offsetWriter.add(offsets[i]);
}
offsetWriter.finish();
+ success = true;
} finally {
- super.finish(docCount);
bytesUsed.addAndGet((-docToEntry.length)
* RamUsageEstimator.NUM_BYTES_INT);
- hash.close();
+ docToEntry = null;
+ IOUtils.closeSafely(!success, idxOut);
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java Thu Jul 7 14:18:02 2011
@@ -25,12 +25,17 @@ import org.apache.lucene.index.values.By
import org.apache.lucene.index.values.Bytes.BytesWriterBase;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PagedBytes;
import org.apache.lucene.util.RamUsageEstimator;
+import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
import org.apache.lucene.util.packed.PackedInts;
+import org.apache.lucene.util.packed.PackedInts.ReaderIterator;
// Variable length byte[] per document, no sharing
@@ -48,18 +53,18 @@ class VarStraightBytesImpl {
// start at -1 if the first added value is > 0
private int lastDocID = -1;
private long[] docToAddress;
-
+ private final ByteBlockPool pool;
+ private IndexOutput datOut;
+ private boolean merge = false;
public Writer(Directory dir, String id, AtomicLong bytesUsed)
throws IOException {
- super(dir, id, CODEC_NAME, VERSION_CURRENT, true, null, bytesUsed);
+ super(dir, id, CODEC_NAME, VERSION_CURRENT, bytesUsed);
+ pool = new ByteBlockPool(new DirectTrackingAllocator(bytesUsed));
docToAddress = new long[1];
+ pool.nextBuffer(); // init
bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT);
}
- public Writer(Directory dir, String id) throws IOException {
- this(dir, id, new AtomicLong());
- }
-
// Fills up to but not including this docID
private void fill(final int docID) {
if (docID >= docToAddress.length) {
@@ -71,21 +76,109 @@ class VarStraightBytesImpl {
for (int i = lastDocID + 1; i < docID; i++) {
docToAddress[i] = address;
}
- lastDocID = docID;
}
@Override
public void add(int docID, BytesRef bytes) throws IOException {
- if (bytes.length == 0)
+ assert !merge;
+ if (bytes.length == 0) {
return; // default
+ }
fill(docID);
docToAddress[docID] = address;
- datOut.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ pool.copy(bytes);
address += bytes.length;
+ lastDocID = docID;
}
+
+ @Override
+ protected void merge(MergeState state) throws IOException {
+ merge = true;
+ datOut = getDataOut();
+ boolean success = false;
+ try {
+ if (state.liveDocs == null && state.reader instanceof Reader) {
+ // bulk merge since we don't have any deletes
+ Reader reader = (Reader) state.reader;
+ final int maxDocs = reader.maxDoc;
+ if (maxDocs == 0) {
+ return;
+ }
+ if (lastDocID+1 < state.docBase) {
+ fill(state.docBase);
+ lastDocID = state.docBase-1;
+ }
+ final long numDataBytes;
+ final IndexInput cloneIdx = reader.cloneIndex();
+ try {
+ numDataBytes = cloneIdx.readVLong();
+ final ReaderIterator iter = PackedInts.getReaderIterator(cloneIdx);
+ for (int i = 0; i < maxDocs; i++) {
+ long offset = iter.next();
+ ++lastDocID;
+ if (lastDocID >= docToAddress.length) {
+ int oldSize = docToAddress.length;
+ docToAddress = ArrayUtil.grow(docToAddress, 1 + lastDocID);
+ bytesUsed.addAndGet((docToAddress.length - oldSize)
+ * RamUsageEstimator.NUM_BYTES_INT);
+ }
+ docToAddress[lastDocID] = address + offset;
+ }
+ address += numDataBytes; // this is the address after all addr pointers are updated
+ iter.close();
+ } finally {
+ IOUtils.closeSafely(true, cloneIdx);
+ }
+ final IndexInput cloneData = reader.cloneData();
+ try {
+ datOut.copyBytes(cloneData, numDataBytes);
+ } finally {
+ IOUtils.closeSafely(true, cloneData);
+ }
+ } else {
+ super.merge(state);
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeSafely(!success, datOut);
+ }
+ }
+ }
+
+ @Override
+ protected void mergeDoc(int docID) throws IOException {
+ assert merge;
+ assert lastDocID < docID;
+ if (bytesRef.length == 0) {
+ return; // default
+ }
+ fill(docID);
+ datOut.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
+ docToAddress[docID] = address;
+ address += bytesRef.length;
+ lastDocID = docID;
+ }
+
@Override
public void finish(int docCount) throws IOException {
+ boolean success = false;
+ assert (!merge && datOut == null) || (merge && datOut != null);
+ final IndexOutput datOut = getDataOut();
+ try {
+ if (!merge) {
+ // header is already written in getDataOut()
+ pool.writePool(datOut);
+ }
+ success = true;
+ } finally {
+ IOUtils.closeSafely(!success, datOut);
+ pool.dropBuffersAndReset();
+ }
+
+ success = false;
+ final IndexOutput idxOut = getIndexOut();
try {
if (lastDocID == -1) {
idxOut.writeVLong(0);
@@ -105,11 +198,12 @@ class VarStraightBytesImpl {
}
w.finish();
}
+ success = true;
} finally {
bytesUsed.addAndGet(-(docToAddress.length)
* RamUsageEstimator.NUM_BYTES_INT);
docToAddress = null;
- super.finish(docCount);
+ IOUtils.closeSafely(!success, idxOut);
}
}
@@ -183,21 +277,23 @@ class VarStraightBytesImpl {
}
private class VarStraightBytesEnum extends ValuesEnum {
- private final PackedInts.Reader addresses;
+ private final PackedInts.ReaderIterator addresses;
private final IndexInput datIn;
private final IndexInput idxIn;
private final long fp;
private final long totBytes;
private int pos = -1;
+ private long nextAddress;
protected VarStraightBytesEnum(AttributeSource source, IndexInput datIn,
IndexInput idxIn) throws IOException {
super(source, ValueType.BYTES_VAR_STRAIGHT);
totBytes = idxIn.readVLong();
fp = datIn.getFilePointer();
- addresses = PackedInts.getReader(idxIn);
+ addresses = PackedInts.getReaderIterator(idxIn);
this.datIn = datIn;
this.idxIn = idxIn;
+ nextAddress = addresses.next();
}
@Override
@@ -211,7 +307,7 @@ class VarStraightBytesImpl {
if (target >= maxDoc) {
return pos = NO_MORE_DOCS;
}
- final long addr = addresses.get(target);
+ final long addr = pos+1 == target ? nextAddress : addresses.advance(target);
if (addr == totBytes) { // empty values at the end
bytesRef.length = 0;
bytesRef.offset = 0;
@@ -219,7 +315,7 @@ class VarStraightBytesImpl {
}
datIn.seek(fp + addr);
final int size = (int) (target == maxDoc - 1 ? totBytes - addr
- : addresses.get(target + 1) - addr);
+ : (nextAddress = addresses.next()) - addr);
if (bytesRef.bytes.length < size) {
bytesRef.grow(size);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Writer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Writer.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Writer.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/index/values/Writer.java Thu Jul 7 14:18:02 2011
@@ -113,11 +113,11 @@ public abstract class Writer extends Doc
* the {@link Writer} implementation. The given document ID must always be
* greater than the previous ID or <tt>0</tt> if called the first time.
*/
- protected abstract void add(int docID) throws IOException;
+ protected abstract void mergeDoc(int docID) throws IOException;
/**
* Sets the next {@link ValuesEnum} to consume values from on calls to
- * {@link #add(int)}
+ * {@link #mergeDoc(int)}
*
* @param valuesEnum
* the next {@link ValuesEnum}, this must not be null
@@ -147,19 +147,19 @@ public abstract class Writer extends Doc
// impl. will get the correct reference for the type
// it supports
int docID = state.docBase;
- final Bits bits = state.bits;
+ final Bits liveDocs = state.liveDocs;
final int docCount = state.docCount;
int currentDocId;
if ((currentDocId = valEnum.advance(0)) != ValuesEnum.NO_MORE_DOCS) {
for (int i = 0; i < docCount; i++) {
- if (bits == null || !bits.get(i)) {
+ if (liveDocs == null || liveDocs.get(i)) {
if (currentDocId < i) {
if ((currentDocId = valEnum.advance(i)) == ValuesEnum.NO_MORE_DOCS) {
break; // advance can jump over default values
}
}
if (currentDocId == i) { // we are on the doc to merge
- add(docID);
+ mergeDoc(docID);
}
++docID;
}
@@ -197,8 +197,12 @@ public abstract class Writer extends Doc
comp = BytesRef.getUTF8SortedAsUnicodeComparator();
}
switch (type) {
- case INTS:
- return Ints.getWriter(directory, id, true, bytesUsed);
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ case VAR_INTS:
+ return Ints.getWriter(directory, id, bytesUsed, type);
case FLOAT_32:
return Floats.getWriter(directory, id, 4, bytesUsed);
case FLOAT_64:
@@ -221,6 +225,7 @@ public abstract class Writer extends Doc
case BYTES_VAR_SORTED:
return Bytes.getWriter(directory, id, Bytes.Mode.SORTED, comp, false,
bytesUsed);
+
default:
throw new IllegalArgumentException("Unknown Values: " + type);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/AutomatonQuery.java Thu Jul 7 14:18:02 2011
@@ -117,7 +117,7 @@ public class AutomatonQuery extends Mult
factory = new TermsEnumFactory() {
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
- return new SingleTermsEnum(terms.iterator(), term.createTerm(singleton));
+ return new SingleTermsEnum(terms.iterator(), new Term(field, singleton));
}
};
} else if (BasicOperations.sameLanguage(automaton, BasicOperations.concatenate(
@@ -126,7 +126,7 @@ public class AutomatonQuery extends Mult
factory = new TermsEnumFactory() {
@Override
protected TermsEnum getTermsEnum(Terms terms, AttributeSource atts) throws IOException {
- return new PrefixTermsEnum(terms.iterator(), term.createTerm(commonPrefix));
+ return new PrefixTermsEnum(terms.iterator(), new Term(field, commonPrefix));
}
};
} else {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingCollector.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingCollector.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingCollector.java Thu Jul 7 14:18:02 2011
@@ -17,22 +17,22 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.util.RamUsageEstimator;
+
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.util.RamUsageEstimator;
-
/**
* Caches all docs, and optionally also scores, coming from
* a search, and is then able to replay them to another
* collector. You specify the max RAM this class may use.
- * Once the collection is done, call {@link #isCached}. If
- * this returns true, you can use {@link #replay} against a
- * new collector. If it returns false, this means too much
- * RAM was required and you must instead re-run the original
- * search.
+ * Once the collection is done, call {@link #isCached}. If
+ * this returns true, you can use {@link #replay(Collector)}
+ * against a new collector. If it returns false, this means
+ * too much RAM was required and you must instead re-run the
+ * original search.
*
* <p><b>NOTE</b>: this class consumes 4 (or 8 bytes, if
* scoring is cached) per collected document. If the result
@@ -105,7 +105,16 @@ public abstract class CachingCollector e
cachedScorer = new CachedScorer();
cachedScores = new ArrayList<float[]>();
- curScores = new float[128];
+ curScores = new float[INITIAL_ARRAY_SIZE];
+ cachedScores.add(curScores);
+ }
+
+ ScoreCachingCollector(Collector other, int maxDocsToCache) {
+ super(other, maxDocsToCache);
+
+ cachedScorer = new CachedScorer();
+ cachedScores = new ArrayList<float[]>();
+ curScores = new float[INITIAL_ARRAY_SIZE];
cachedScores.add(curScores);
}
@@ -210,7 +219,11 @@ public abstract class CachingCollector e
NoScoreCachingCollector(Collector other, double maxRAMMB) {
super(other, maxRAMMB, false);
}
-
+
+ NoScoreCachingCollector(Collector other, int maxDocsToCache) {
+ super(other, maxDocsToCache);
+ }
+
@Override
public void collect(int doc) throws IOException {
@@ -353,7 +366,25 @@ public abstract class CachingCollector e
*/
public static CachingCollector create(Collector other, boolean cacheScores, double maxRAMMB) {
return cacheScores ? new ScoreCachingCollector(other, maxRAMMB) : new NoScoreCachingCollector(other, maxRAMMB);
- }
+ }
+
+ /**
+ * Create a new {@link CachingCollector} that wraps the given collector and
+ * caches documents and scores up to the specified max docs threshold.
+ *
+ * @param other
+ * the Collector to wrap and delegate calls to.
+ * @param cacheScores
+ * whether to cache scores in addition to document IDs. Note that
+ * this increases the RAM consumed per doc
+ * @param maxDocsToCache
+ * the maximum number of documents for caching the documents and
+ * possible the scores. If the collector exceeds the threshold,
+ * no documents and scores are cached.
+ */
+ public static CachingCollector create(Collector other, boolean cacheScores, int maxDocsToCache) {
+ return cacheScores ? new ScoreCachingCollector(other, maxDocsToCache) : new NoScoreCachingCollector(other, maxDocsToCache);
+ }
// Prevent extension from non-internal classes
private CachingCollector(Collector other, double maxRAMMB, boolean cacheScores) {
@@ -369,6 +400,15 @@ public abstract class CachingCollector e
}
maxDocsToCache = (int) ((maxRAMMB * 1024 * 1024) / bytesPerDoc);
}
+
+ private CachingCollector(Collector other, int maxDocsToCache) {
+ this.other = other;
+
+ cachedDocs = new ArrayList<int[]>();
+ curDocs = new int[INITIAL_ARRAY_SIZE];
+ cachedDocs.add(curDocs);
+ this.maxDocsToCache = maxDocsToCache;
+ }
@Override
public boolean acceptsDocsOutOfOrder() {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java Thu Jul 7 14:18:02 2011
@@ -54,7 +54,7 @@ public class CachingSpanFilter extends S
}
this.cache = new CachingWrapperFilter.FilterCache<SpanFilterResult>(deletesMode) {
@Override
- protected SpanFilterResult mergeDeletes(final Bits delDocs, final SpanFilterResult value) {
+ protected SpanFilterResult mergeLiveDocs(final Bits liveDocs, final SpanFilterResult value) {
throw new IllegalStateException("DeletesMode.DYNAMIC is not supported");
}
};
@@ -73,7 +73,7 @@ public class CachingSpanFilter extends S
final IndexReader reader = context.reader;
final Object coreKey = reader.getCoreCacheKey();
- final Object delCoreKey = reader.hasDeletions() ? reader.getDeletedDocs() : coreKey;
+ final Object delCoreKey = reader.hasDeletions() ? reader.getLiveDocs() : coreKey;
SpanFilterResult result = cache.get(reader, coreKey, delCoreKey);
if (result != null) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java Thu Jul 7 14:18:02 2011
@@ -103,13 +103,13 @@ public class CachingWrapperFilter extend
value = cache.get(delCoreKey);
if (value == null) {
- // now for core match, but dynamically AND NOT
- // deletions
+ // now for core match, but dynamically AND
+ // live docs
value = cache.get(coreKey);
if (value != null) {
- final Bits delDocs = reader.getDeletedDocs();
- if (delDocs != null) {
- value = mergeDeletes(delDocs, value);
+ final Bits liveDocs = reader.getLiveDocs();
+ if (liveDocs != null) {
+ value = mergeLiveDocs(liveDocs, value);
}
}
}
@@ -118,7 +118,7 @@ public class CachingWrapperFilter extend
return value;
}
- protected abstract T mergeDeletes(Bits delDocs, T value);
+ protected abstract T mergeLiveDocs(Bits liveDocs, T value);
public synchronized void put(Object coreKey, Object delCoreKey, T value) {
if (deletesMode == DeletesMode.IGNORE) {
@@ -158,11 +158,11 @@ public class CachingWrapperFilter extend
this.filter = filter;
cache = new FilterCache<DocIdSet>(deletesMode) {
@Override
- public DocIdSet mergeDeletes(final Bits delDocs, final DocIdSet docIdSet) {
+ public DocIdSet mergeLiveDocs(final Bits liveDocs, final DocIdSet docIdSet) {
return new FilteredDocIdSet(docIdSet) {
@Override
- protected boolean match(int docID) {
- return !delDocs.get(docID);
+ protected boolean match(int docID) {
+ return liveDocs.get(docID);
}
};
}
@@ -197,7 +197,7 @@ public class CachingWrapperFilter extend
public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException {
final IndexReader reader = context.reader;
final Object coreKey = reader.getCoreCacheKey();
- final Object delCoreKey = reader.hasDeletions() ? reader.getDeletedDocs() : coreKey;
+ final Object delCoreKey = reader.hasDeletions() ? reader.getLiveDocs() : coreKey;
DocIdSet docIdSet = cache.get(reader, coreKey, delCoreKey);
if (docIdSet != null) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java Thu Jul 7 14:18:02 2011
@@ -100,14 +100,13 @@ class ConstantScoreAutoRewrite extends T
return getTopLevelQuery();
} else {
final BooleanQuery bq = getTopLevelQuery();
- final Term placeholderTerm = new Term(query.field);
final BytesRefHash pendingTerms = col.pendingTerms;
final int sort[] = pendingTerms.sort(col.termsEnum.getComparator());
for(int i = 0; i < size; i++) {
final int pos = sort[i];
// docFreq is not used for constant score here, we pass 1
// to explicitely set a fake value, so it's not calculated
- addClause(bq, placeholderTerm.createTerm(pendingTerms.get(pos, new BytesRef())), 1, 1.0f, col.array.termState[pos]);
+ addClause(bq, new Term(query.field, pendingTerms.get(pos, new BytesRef())), 1, 1.0f, col.array.termState[pos]);
}
// Strip scores
final Query result = new ConstantScoreQuery(bq);
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Thu Jul 7 14:18:02 2011
@@ -131,16 +131,18 @@ public class ConstantScoreQuery extends
if (filter != null) {
assert query == null;
final DocIdSet dis = filter.getDocIdSet(context);
- if (dis == null)
+ if (dis == null) {
return null;
+ }
disi = dis.iterator();
} else {
assert query != null && innerWeight != null;
- disi =
- innerWeight.scorer(context, scorerContext);
+ disi = innerWeight.scorer(context, scorerContext);
}
- if (disi == null)
+
+ if (disi == null) {
return null;
+ }
return new ConstantScorer(disi, this, queryWeight);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Thu Jul 7 14:18:02 2011
@@ -42,7 +42,6 @@ import org.apache.lucene.search.cache.Ca
import org.apache.lucene.search.cache.CachedArray.LongValues;
import org.apache.lucene.search.cache.CachedArray.ShortValues;
import org.apache.lucene.util.FieldCacheSanityChecker;
-import org.apache.lucene.util.StringHelper;
/**
* Expert: The default cache implementation, storing all values in memory.
@@ -138,7 +137,7 @@ public class FieldCacheImpl implements F
}
final static IndexReader.ReaderFinishedListener purgeReader = new IndexReader.ReaderFinishedListener() {
- // @Override -- not until Java 1.6
+ @Override
public void finished(IndexReader reader) {
FieldCache.DEFAULT.purge(reader);
}
@@ -246,7 +245,7 @@ public class FieldCacheImpl implements F
/** Creates one of these objects for a custom comparator/parser. */
Entry (String field, EntryCreator<T> custom) {
- this.field = StringHelper.intern(field);
+ this.field = field;
this.creator = custom;
}
@@ -255,7 +254,7 @@ public class FieldCacheImpl implements F
public boolean equals (Object o) {
if (o instanceof Entry) {
Entry other = (Entry) o;
- if (other.field == field) {
+ if (other.field.equals(field)) {
if (other.creator == null) {
if (creator == null) return true;
} else if (other.creator.equals (creator)) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Thu Jul 7 14:18:02 2011
@@ -20,7 +20,6 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -533,9 +532,9 @@ public abstract class FieldCacheRangeFil
@Override
public DocIdSetIterator iterator() throws IOException {
- final Bits skipDocs = canIgnoreDeletedDocs ? null : reader.getDeletedDocs();
+ final Bits liveDocs = canIgnoreDeletedDocs ? null : reader.getLiveDocs();
- if (skipDocs == null) {
+ if (liveDocs == null) {
// Specialization optimization disregard deletions
return new DocIdSetIterator() {
private int doc = -1;
@@ -575,7 +574,7 @@ public abstract class FieldCacheRangeFil
final int maxDoc = reader.maxDoc();
// a DocIdSetIterator generating docIds by
- // incrementing a variable & checking skipDocs -
+ // incrementing a variable & checking liveDocs -
return new DocIdSetIterator() {
private int doc = -1;
@Override
@@ -590,14 +589,14 @@ public abstract class FieldCacheRangeFil
if (doc >= maxDoc) {
return doc = NO_MORE_DOCS;
}
- } while (skipDocs.get(doc) || !matchDoc(doc));
+ } while (!liveDocs.get(doc) || !matchDoc(doc));
return doc;
}
@Override
public int advance(int target) {
for(doc=target;doc<maxDoc;doc++) {
- if (!skipDocs.get(doc) && matchDoc(doc)) {
+ if (liveDocs.get(doc) && matchDoc(doc)) {
return doc;
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/search/FieldComparator.java Thu Jul 7 14:18:02 2011
@@ -793,8 +793,13 @@ public abstract class FieldComparator<T>
@Override
public void setScorer(Scorer scorer) {
// wrap with a ScoreCachingWrappingScorer so that successive calls to
- // score() will not incur score computation over and over again.
- this.scorer = new ScoreCachingWrappingScorer(scorer);
+ // score() will not incur score computation over and
+ // over again.
+ if (!(scorer instanceof ScoreCachingWrappingScorer)) {
+ this.scorer = new ScoreCachingWrappingScorer(scorer);
+ } else {
+ this.scorer = scorer;
+ }
}
@Override