You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/07/07 16:18:33 UTC
svn commit: r1143840 [10/22] - in /lucene/dev/branches/flexscoring: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/
dev-tools/idea/lucene/contrib/highlighter/ dev-tools/idea/lucene/contrib/...
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java Thu Jul 7 14:18:02 2011
@@ -21,6 +21,8 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import org.apache.lucene.util.IOUtils;
+
/** A straightforward implementation of {@link FSDirectory}
* using java.io.RandomAccessFile. However, this class has
* poor concurrent performance (multiple threads will
@@ -55,6 +57,45 @@ public class SimpleFSDirectory extends F
ensureOpen();
return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize());
}
+
+ @Override
+ public CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ return new SimpleFSCompoundFileDirectory(name, bufferSize);
+ }
+
+ private final class SimpleFSCompoundFileDirectory extends CompoundFileDirectory {
+ private SimpleFSIndexInput.Descriptor fd;
+
+ public SimpleFSCompoundFileDirectory(String fileName, int readBufferSize) throws IOException {
+ super(SimpleFSDirectory.this, fileName, readBufferSize);
+ IndexInput stream = null;
+ try {
+ final File f = new File(SimpleFSDirectory.this.getDirectory(), fileName);
+ fd = new SimpleFSIndexInput.Descriptor(f, "r");
+ stream = new SimpleFSIndexInput(fd, 0, fd.length, readBufferSize,
+ getReadChunkSize());
+ initForRead(CompoundFileDirectory.readEntries(stream, SimpleFSDirectory.this, fileName));
+ stream.close();
+ } catch (IOException e) {
+ // throw our original exception
+ IOUtils.closeSafely(e, fd, stream);
+ }
+ }
+
+ @Override
+ public IndexInput openInputSlice(String id, long offset, long length, int readBufferSize) throws IOException {
+ return new SimpleFSIndexInput(fd, offset, length, readBufferSize, getReadChunkSize());
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ try {
+ fd.close();
+ } finally {
+ super.close();
+ }
+ }
+ }
protected static class SimpleFSIndexInput extends BufferedIndexInput {
@@ -84,11 +125,24 @@ public class SimpleFSDirectory extends F
boolean isClone;
// LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM
protected final int chunkSize;
+ protected final long off;
+ protected final long end;
public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException {
super(bufferSize);
- file = new Descriptor(path, "r");
+ this.file = new Descriptor(path, "r");
this.chunkSize = chunkSize;
+ this.off = 0L;
+ this.end = file.length;
+ }
+
+ public SimpleFSIndexInput(Descriptor file, long off, long length, int bufferSize, int chunkSize) throws IOException {
+ super(bufferSize);
+ this.file = file;
+ this.chunkSize = chunkSize;
+ this.off = off;
+ this.end = off + length;
+ this.isClone = true; // well, we are sorta?
}
/** IndexInput methods */
@@ -96,13 +150,17 @@ public class SimpleFSDirectory extends F
protected void readInternal(byte[] b, int offset, int len)
throws IOException {
synchronized (file) {
- long position = getFilePointer();
+ long position = off + getFilePointer();
if (position != file.position) {
file.seek(position);
file.position = position;
}
int total = 0;
+ if (position + len > end) {
+ throw new IOException("read past EOF");
+ }
+
try {
do {
final int readLength;
@@ -113,9 +171,6 @@ public class SimpleFSDirectory extends F
readLength = chunkSize;
}
final int i = file.read(b, offset + total, readLength);
- if (i == -1) {
- throw new IOException("read past EOF");
- }
file.position += i;
total += i;
} while (total < len);
@@ -144,7 +199,7 @@ public class SimpleFSDirectory extends F
@Override
public long length() {
- return file.length;
+ return end - off;
}
@Override
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ArrayUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ArrayUtil.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ArrayUtil.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ArrayUtil.java Thu Jul 7 14:18:02 2011
@@ -393,6 +393,56 @@ public final class ArrayUtil {
return array;
}
+ public static int[][] grow(int[][] array, int minSize) {
+ if (array.length < minSize) {
+ int[][] newArray = new int[oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
+ System.arraycopy(array, 0, newArray, 0, array.length);
+ return newArray;
+ } else {
+ return array;
+ }
+ }
+
+ public static int[][] grow(int[][] array) {
+ return grow(array, 1 + array.length);
+ }
+
+ public static int[][] shrink(int[][] array, int targetSize) {
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
+ if (newSize != array.length) {
+ int[][] newArray = new int[newSize][];
+ System.arraycopy(array, 0, newArray, 0, newSize);
+ return newArray;
+ } else {
+ return array;
+ }
+ }
+
+ public static float[][] grow(float[][] array, int minSize) {
+ if (array.length < minSize) {
+ float[][] newArray = new float[oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF)][];
+ System.arraycopy(array, 0, newArray, 0, array.length);
+ return newArray;
+ } else {
+ return array;
+ }
+ }
+
+ public static float[][] grow(float[][] array) {
+ return grow(array, 1 + array.length);
+ }
+
+ public static float[][] shrink(float[][] array, int targetSize) {
+ final int newSize = getShrinkSize(array.length, targetSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
+ if (newSize != array.length) {
+ float[][] newArray = new float[newSize][];
+ System.arraycopy(array, 0, newArray, 0, newSize);
+ return newArray;
+ } else {
+ return array;
+ }
+ }
+
/**
* Returns hash of chars in range start (inclusive) to
* end (inclusive)
@@ -617,6 +667,7 @@ public final class ArrayUtil {
*/
public static <T> void mergeSort(T[] a, int fromIndex, int toIndex, Comparator<? super T> comp) {
if (toIndex-fromIndex <= 1) return;
+ //System.out.println("SORT: " + (toIndex-fromIndex));
getSorter(a, comp).mergeSort(fromIndex, toIndex-1);
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitUtil.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitUtil.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitUtil.java Thu Jul 7 14:18:02 2011
@@ -778,6 +778,28 @@ public final class BitUtil {
return n - (y & 1);
}
+ /** table of number of leading zeros in a byte */
+ public static final byte[] nlzTable = {8,7,6,6,5,5,5,5,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0};
+
+ /** Returns the number of leading zero bits.
+ */
+ public static int nlz(long x) {
+ int n = 0;
+ // do the first step as a long
+ int y = (int)(x>>>32);
+ if (y==0) {n+=32; y = (int)(x); }
+ if ((y & 0xFFFF0000) == 0) { n+=16; y<<=16; }
+ if ((y & 0xFF000000) == 0) { n+=8; y<<=8; }
+ return n + nlzTable[y >>> 24];
+ /* implementation without table:
+ if ((y & 0xF0000000) == 0) { n+=4; y<<=4; }
+ if ((y & 0xC0000000) == 0) { n+=2; y<<=2; }
+ if ((y & 0x80000000) == 0) { n+=1; y<<=1; }
+ if ((y & 0x80000000) == 0) { n+=1;}
+ return n;
+ */
+ }
+
/** returns true if v is a power of two or zero*/
public static boolean isPowerOfTwo(int v) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitVector.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitVector.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BitVector.java Thu Jul 7 14:18:02 2011
@@ -18,39 +18,51 @@ package org.apache.lucene.util;
*/
import java.io.IOException;
+import java.util.Arrays;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
/** Optimized implementation of a vector of bits. This is more-or-less like
- java.util.BitSet, but also includes the following:
- <ul>
- <li>a count() method, which efficiently computes the number of one bits;</li>
- <li>optimized read from and write to disk;</li>
- <li>inlinable get() method;</li>
- <li>store and load, as bit set or d-gaps, depending on sparseness;</li>
- </ul>
- */
+ * java.util.BitSet, but also includes the following:
+ * <ul>
+ * <li>a count() method, which efficiently computes the number of one bits;</li>
+ * <li>optimized read from and write to disk;</li>
+ * <li>inlinable get() method;</li>
+ * <li>store and load, as bit set or d-gaps, depending on sparseness;</li>
+ * </ul>
+ *
+ * @lucene.internal
+ */
public final class BitVector implements Cloneable, Bits {
private byte[] bits;
private int size;
private int count;
+ private int version;
/** Constructs a vector capable of holding <code>n</code> bits. */
public BitVector(int n) {
size = n;
- bits = new byte[(size >> 3) + 1];
+ bits = new byte[getNumBytes(size)];
count = 0;
}
-
+
BitVector(byte[] bits, int size) {
this.bits = bits;
this.size = size;
count = -1;
}
+ private int getNumBytes(int size) {
+ int bytesLength = size >>> 3;
+ if ((size & 7) != 0) {
+ bytesLength++;
+ }
+ return bytesLength;
+ }
+
@Override
public Object clone() {
byte[] copyBits = new byte[bits.length];
@@ -82,8 +94,10 @@ public final class BitVector implements
return true;
else {
bits[pos] = (byte) (v | flag);
- if (count != -1)
+ if (count != -1) {
count++;
+ assert count <= size;
+ }
return false;
}
}
@@ -97,6 +111,25 @@ public final class BitVector implements
count = -1;
}
+ public final boolean getAndClear(int bit) {
+ if (bit >= size) {
+ throw new ArrayIndexOutOfBoundsException(bit);
+ }
+ final int pos = bit >> 3;
+ final int v = bits[pos];
+ final int flag = 1 << (bit & 7);
+ if ((flag & v) == 0) {
+ return false;
+ } else {
+ bits[pos] &= ~flag;
+ if (count != -1) {
+ count--;
+ assert count >= 0;
+ }
+ return true;
+ }
+ }
+
/** Returns <code>true</code> if <code>bit</code> is one and
<code>false</code> if it is zero. */
public final boolean get(int bit) {
@@ -110,7 +143,7 @@ public final class BitVector implements
return size;
}
- // @Override -- not until Java 1.6
+ @Override
public int length() {
return size;
}
@@ -123,8 +156,9 @@ public final class BitVector implements
if (count == -1) {
int c = 0;
int end = bits.length;
- for (int i = 0; i < end; i++)
+ for (int i = 0; i < end; i++) {
c += BYTE_COUNTS[bits[i] & 0xFF]; // sum bits per byte
+ }
count = c;
}
return count;
@@ -134,8 +168,9 @@ public final class BitVector implements
public final int getRecomputedCount() {
int c = 0;
int end = bits.length;
- for (int i = 0; i < end; i++)
+ for (int i = 0; i < end; i++) {
c += BYTE_COUNTS[bits[i] & 0xFF]; // sum bits per byte
+ }
return c;
}
@@ -158,6 +193,24 @@ public final class BitVector implements
4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
};
+ private static String CODEC = "BitVector";
+
+ // Version before version tracking was added:
+ public final static int VERSION_PRE = -1;
+
+ // First version:
+ public final static int VERSION_START = 0;
+
+ // Changed DGaps to encode gaps between cleared bits, not
+ // set:
+ public final static int VERSION_DGAPS_CLEARED = 1;
+
+ // Increment version to change it:
+ public final static int VERSION_CURRENT = VERSION_DGAPS_CLEARED;
+
+ public int getVersion() {
+ return version;
+ }
/** Writes this vector to the file <code>name</code> in Directory
<code>d</code>, in a format that can be read by the constructor {@link
@@ -165,8 +218,11 @@ public final class BitVector implements
public final void write(Directory d, String name) throws IOException {
IndexOutput output = d.createOutput(name);
try {
+ output.writeInt(-2);
+ CodecUtil.writeHeader(output, CODEC, VERSION_CURRENT);
if (isSparse()) {
- writeDgaps(output); // sparse bit-set more efficiently saved as d-gaps.
+ // sparse bit-set more efficiently saved as d-gaps.
+ writeClearedDgaps(output);
} else {
writeBits(output);
}
@@ -174,6 +230,38 @@ public final class BitVector implements
output.close();
}
}
+
+ /** Invert all bits */
+ public void invertAll() {
+ if (count != -1) {
+ count = size - count;
+ }
+ if (bits.length > 0) {
+ for(int idx=0;idx<bits.length;idx++) {
+ bits[idx] = (byte) (~bits[idx]);
+ }
+ clearUnusedBits();
+ }
+ }
+
+ private void clearUnusedBits() {
+ // Take care not to invert the "unused" bits in the
+ // last byte:
+ if (bits.length > 0) {
+ final int lastNBits = size & 7;
+ if (lastNBits != 0) {
+ final int mask = (1 << lastNBits)-1;
+ bits[bits.length-1] &= mask;
+ }
+ }
+ }
+
+ /** Set all bits */
+ public void setAll() {
+ Arrays.fill(bits, (byte) 0xff);
+ clearUnusedBits();
+ count = size;
+ }
/** Write as a bit set */
private void writeBits(IndexOutput output) throws IOException {
@@ -183,38 +271,58 @@ public final class BitVector implements
}
/** Write as a d-gaps list */
- private void writeDgaps(IndexOutput output) throws IOException {
+ private void writeClearedDgaps(IndexOutput output) throws IOException {
output.writeInt(-1); // mark using d-gaps
output.writeInt(size()); // write size
output.writeInt(count()); // write count
int last=0;
- int n = count();
+ int numCleared = size()-count();
int m = bits.length;
- for (int i=0; i<m && n>0; i++) {
- if (bits[i]!=0) {
+ for (int i=0; i<m && numCleared>0; i++) {
+ if (bits[i]!=0xff) {
output.writeVInt(i-last);
output.writeByte(bits[i]);
last = i;
- n -= BYTE_COUNTS[bits[i] & 0xFF];
+ numCleared -= (8-BYTE_COUNTS[bits[i] & 0xFF]);
+ assert numCleared >= 0;
}
}
}
/** Indicates if the bit vector is sparse and should be saved as a d-gaps list, or dense, and should be saved as a bit set. */
private boolean isSparse() {
- // note: order of comparisons below set to favor smaller values (no binary range search.)
- // note: adding 4 because we start with ((int) -1) to indicate d-gaps format.
- // note: we write the d-gap for the byte number, and the byte (bits[i]) itself, therefore
- // multiplying count by (8+8) or (8+16) or (8+24) etc.:
- // - first 8 for writing bits[i] (1 byte vs. 1 bit), and
- // - second part for writing the byte-number d-gap as vint.
+
+ final int clearedCount = size() - count();
+ if (clearedCount == 0) {
+ return true;
+ }
+
+ final int avgGapLength = bits.length / clearedCount;
+
+ // expected number of bytes for vInt encoding of each gap
+ final int expectedDGapBytes;
+ if (avgGapLength <= (1<< 7)) {
+ expectedDGapBytes = 1;
+ } else if (avgGapLength <= (1<<14)) {
+ expectedDGapBytes = 2;
+ } else if (avgGapLength <= (1<<21)) {
+ expectedDGapBytes = 3;
+ } else if (avgGapLength <= (1<<28)) {
+ expectedDGapBytes = 4;
+ } else {
+ expectedDGapBytes = 5;
+ }
+
+ // +1 because we write the byte itself that contains the
+ // set bit
+ final int bytesPerSetBit = expectedDGapBytes + 1;
+
+ // note: adding 32 because we start with ((int) -1) to indicate d-gaps format.
+ final long expectedBits = 32 + 8 * bytesPerSetBit * count();
+
// note: factor is for read/write of byte-arrays being faster than vints.
- int factor = 10;
- if (bits.length < (1<< 7)) return factor * (4 + (8+ 8)*count()) < size();
- if (bits.length < (1<<14)) return factor * (4 + (8+16)*count()) < size();
- if (bits.length < (1<<21)) return factor * (4 + (8+24)*count()) < size();
- if (bits.length < (1<<28)) return factor * (4 + (8+32)*count()) < size();
- return factor * (4 + (8+40)*count()) < size();
+ final long factor = 10;
+ return factor * expectedBits < size();
}
/** Constructs a bit vector from the file <code>name</code> in Directory
@@ -222,10 +330,24 @@ public final class BitVector implements
*/
public BitVector(Directory d, String name) throws IOException {
IndexInput input = d.openInput(name);
+
try {
- size = input.readInt(); // read size
+ final int firstInt = input.readInt();
+
+ if (firstInt == -2) {
+ // New format, with full header & version:
+ version = CodecUtil.checkHeader(input, CODEC, VERSION_START, VERSION_CURRENT);
+ size = input.readInt();
+ } else {
+ version = VERSION_PRE;
+ size = firstInt;
+ }
if (size == -1) {
- readDgaps(input);
+ if (version >= VERSION_DGAPS_CLEARED) {
+ readClearedDgaps(input);
+ } else {
+ readSetDgaps(input);
+ }
} else {
readBits(input);
}
@@ -237,12 +359,12 @@ public final class BitVector implements
/** Read as a bit set */
private void readBits(IndexInput input) throws IOException {
count = input.readInt(); // read count
- bits = new byte[(size >> 3) + 1]; // allocate bits
+ bits = new byte[getNumBytes(size)]; // allocate bits
input.readBytes(bits, 0, bits.length);
}
/** read as a d-gaps list */
- private void readDgaps(IndexInput input) throws IOException {
+ private void readSetDgaps(IndexInput input) throws IOException {
size = input.readInt(); // (re)read size
count = input.readInt(); // read count
bits = new byte[(size >> 3) + 1]; // allocate bits
@@ -252,32 +374,24 @@ public final class BitVector implements
last += input.readVInt();
bits[last] = input.readByte();
n -= BYTE_COUNTS[bits[last] & 0xFF];
+ assert n >= 0;
}
}
- /**
- * Retrieve a subset of this BitVector.
- *
- * @param start
- * starting index, inclusive
- * @param end
- * ending index, exclusive
- * @return subset
- */
- public BitVector subset(int start, int end) {
- if (start < 0 || end > size() || end < start)
- throw new IndexOutOfBoundsException();
- // Special case -- return empty vector is start == end
- if (end == start) return new BitVector(0);
- byte[] bits = new byte[((end - start - 1) >>> 3) + 1];
- int s = start >>> 3;
- for (int i = 0; i < bits.length; i++) {
- int cur = 0xFF & this.bits[i + s];
- int next = i + s + 1 >= this.bits.length ? 0 : 0xFF & this.bits[i + s + 1];
- bits[i] = (byte) ((cur >>> (start & 7)) | ((next << (8 - (start & 7)))));
- }
- int bitsToClear = (bits.length * 8 - (end - start)) % 8;
- bits[bits.length - 1] &= ~(0xFF << (8 - bitsToClear));
- return new BitVector(bits, end - start);
+ /** read as a d-gaps cleared bits list */
+ private void readClearedDgaps(IndexInput input) throws IOException {
+ size = input.readInt(); // (re)read size
+ count = input.readInt(); // read count
+ bits = new byte[(size >> 3) + 1]; // allocate bits
+ Arrays.fill(bits, (byte) 0xff);
+ clearUnusedBits();
+ int last=0;
+ int numCleared = size()-count();
+ while (numCleared>0) {
+ last += input.readVInt();
+ bits[last] = input.readByte();
+ numCleared -= 8-BYTE_COUNTS[bits[last] & 0xFF];
+ assert numCleared >= 0;
+ }
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/ByteBlockPool.java Thu Jul 7 14:18:02 2011
@@ -16,10 +16,13 @@ package org.apache.lucene.util;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.lucene.store.DataOutput;
+
import static org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF;
/**
@@ -241,5 +244,42 @@ public final class ByteBlockPool {
assert term.length >= 0;
return term;
}
+
+ /**
+ * Copies the given {@link BytesRef} at the current positions (
+ * {@link #byteUpto} across buffer boundaries
+ */
+ public final void copy(final BytesRef bytes) {
+ int length = bytes.length;
+ int offset = bytes.offset;
+ int overflow = (length + byteUpto) - BYTE_BLOCK_SIZE;
+ do {
+ if (overflow <= 0) {
+ System.arraycopy(bytes.bytes, offset, buffer, byteUpto, length);
+ byteUpto += length;
+ break;
+ } else {
+ final int bytesToCopy = length-overflow;
+ System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy);
+ offset += bytesToCopy;
+ length -= bytesToCopy;
+ nextBuffer();
+ overflow = overflow - BYTE_BLOCK_SIZE;
+ }
+ } while(true);
+ }
+
+ /**
+ * Writes the pools content to the given {@link DataOutput}
+ */
+ public final void writePool(final DataOutput out) throws IOException {
+ int bytesOffset = byteOffset;
+ int block = 0;
+ while (bytesOffset > 0) {
+ out.writeBytes(buffers[block++], BYTE_BLOCK_SIZE);
+ bytesOffset -= BYTE_BLOCK_SIZE;
+ }
+ out.writeBytes(buffers[block], byteUpto);
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BytesRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BytesRef.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BytesRef.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/BytesRef.java Thu Jul 7 14:18:02 2011
@@ -20,7 +20,8 @@ package org.apache.lucene.util;
import java.util.Comparator;
/** Represents byte[], as a slice (offset + length) into an
- * existing byte[].
+ * existing byte[]. The {@link #bytes} member should never be null;
+ * use {@link #EMPTY_BYTES} if necessary.
*
* @lucene.experimental */
public final class BytesRef implements Comparable<BytesRef> {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/CharsRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/CharsRef.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/CharsRef.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/CharsRef.java Thu Jul 7 14:18:02 2011
@@ -19,7 +19,8 @@ package org.apache.lucene.util;
/**
* Represents char[], as a slice (offset + length) into an existing char[].
- *
+ * The {@link #chars} member should never be null; use
+ * {@link #EMPTY_ARRAY} if necessary.
* @lucene.internal
*/
public final class CharsRef implements Comparable<CharsRef>, CharSequence {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/FieldCacheSanityChecker.java Thu Jul 7 14:18:02 2011
@@ -208,7 +208,7 @@ public final class FieldCacheSanityCheck
if (seen.contains(rf)) continue;
- List<Object> kids = getAllDecendentReaderKeys(rf.readerKey);
+ List<Object> kids = getAllDescendentReaderKeys(rf.readerKey);
for (Object kidKey : kids) {
ReaderField kid = new ReaderField(kidKey, rf.fieldName);
@@ -252,7 +252,7 @@ public final class FieldCacheSanityCheck
badness = badEntries.toArray(badness);
insanity.add(new Insanity(InsanityType.SUBREADER,
- "Found caches for decendents of " +
+ "Found caches for descendants of " +
parent.toString(),
badness));
}
@@ -266,7 +266,7 @@ public final class FieldCacheSanityCheck
* the hierarchy of subReaders building up a list of the objects
* returned by obj.getFieldCacheKey()
*/
- private List<Object> getAllDecendentReaderKeys(Object seed) {
+ private List<Object> getAllDescendentReaderKeys(Object seed) {
List<Object> all = new ArrayList<Object>(17); // will grow as we iter
all.add(seed);
for (int i = 0; i < all.size(); i++) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/IntsRef.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/IntsRef.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/IntsRef.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/IntsRef.java Thu Jul 7 14:18:02 2011
@@ -18,11 +18,14 @@ package org.apache.lucene.util;
*/
/** Represents int[], as a slice (offset + length) into an
- * existing int[].
+ * existing int[]. The {@link #ints} member should never be null; use
+ * {@link #EMPTY_INTS} if necessary.
*
* @lucene.internal */
public final class IntsRef implements Comparable<IntsRef> {
+ public static final int[] EMPTY_INTS = new int[0];
+
public int[] ints;
public int offset;
public int length;
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/MultiBits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/MultiBits.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/MultiBits.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/MultiBits.java Thu Jul 7 14:18:02 2011
@@ -34,13 +34,16 @@ public final class MultiBits implements
// length is 1+subs.length (the last entry has the maxDoc):
private final int[] starts;
- public MultiBits(List<Bits> bits, List<Integer> starts) {
+ private final boolean defaultValue;
+
+ public MultiBits(List<Bits> bits, List<Integer> starts, boolean defaultValue) {
assert starts.size() == 1+bits.size();
this.subs = bits.toArray(Bits.EMPTY_ARRAY);
this.starts = new int[starts.size()];
for(int i=0;i<this.starts.length;i++) {
this.starts[i] = starts.get(i);
}
+ this.defaultValue = defaultValue;
}
private boolean checkLength(int reader, int doc) {
@@ -54,7 +57,7 @@ public final class MultiBits implements
assert reader != -1;
final Bits bits = subs[reader];
if (bits == null) {
- return false;
+ return defaultValue;
} else {
assert checkLength(reader, doc);
return bits.get(doc-starts[reader]);
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/OpenBitSet.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/OpenBitSet.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/OpenBitSet.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/OpenBitSet.java Thu Jul 7 14:18:02 2011
@@ -133,10 +133,10 @@ public class OpenBitSet extends DocIdSet
* compatibility. This is *not* equal to {@link #cardinality}
*/
public long size() {
- return capacity();
+ return capacity();
}
- // @Override -- not until Java 1.6
+ @Override
public int length() {
return bits.length << 6;
}
@@ -659,7 +659,71 @@ public class OpenBitSet extends DocIdSet
}
+ /** Returns the index of the first set bit starting downwards at
+ * the index specified.
+ * -1 is returned if there are no more set bits.
+ */
+ public int prevSetBit(int index) {
+ int i = index >> 6;
+ final int subIndex;
+ long word;
+ if (i >= wlen) {
+ i = wlen - 1;
+ if (i < 0) return -1;
+ subIndex = 63; // last possible bit
+ word = bits[i];
+ } else {
+ if (i < 0) return -1;
+ subIndex = index & 0x3f; // index within the word
+ word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index
+ }
+
+ if (word != 0) {
+ return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
+ }
+
+ while (--i >= 0) {
+ word = bits[i];
+ if (word !=0 ) {
+ return (i << 6) + 63 - Long.numberOfLeadingZeros(word);
+ }
+ }
+
+ return -1;
+ }
+
+ /** Returns the index of the first set bit starting downwards at
+ * the index specified.
+ * -1 is returned if there are no more set bits.
+ */
+ public long prevSetBit(long index) {
+ int i = (int) (index >> 6);
+ final int subIndex;
+ long word;
+ if (i >= wlen) {
+ i = wlen - 1;
+ if (i < 0) return -1;
+ subIndex = 63; // last possible bit
+ word = bits[i];
+ } else {
+ if (i < 0) return -1;
+ subIndex = (int)index & 0x3f; // index within the word
+ word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index
+ }
+
+ if (word != 0) {
+ return (((long)i)<<6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197
+ }
+
+ while (--i >= 0) {
+ word = bits[i];
+ if (word !=0 ) {
+ return (((long)i)<<6) + 63 - Long.numberOfLeadingZeros(word);
+ }
+ }
+ return -1;
+ }
@Override
public Object clone() {
@@ -838,7 +902,6 @@ public class OpenBitSet extends DocIdSet
// empty sets from returning 0, which is too common.
return (int)((h>>32) ^ h) + 0x98761234;
}
-
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/StringHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/StringHelper.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/StringHelper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/StringHelper.java Thu Jul 7 14:18:02 2011
@@ -27,17 +27,6 @@ import java.util.StringTokenizer;
* @lucene.internal
*/
public abstract class StringHelper {
- /**
- * Expert:
- * The StringInterner implementation used by Lucene.
- * This shouldn't be changed to an incompatible implementation after other Lucene APIs have been used.
- */
- public static StringInterner interner = new SimpleStringInterner(1024,8);
-
- /** Return the same string object for all equal strings */
- public static String intern(String s) {
- return interner.intern(s);
- }
/**
* Compares two byte[] arrays, element by element, and returns the
@@ -76,7 +65,7 @@ public abstract class StringHelper {
if (bTokens.hasMoreTokens()) {
int bToken = Integer.parseInt(bTokens.nextToken());
if (aToken != bToken) {
- return aToken - bToken;
+ return aToken < bToken ? -1 : 1;
}
} else {
// a has some extra trailing tokens. if these are all zeroes, thats ok.
@@ -95,4 +84,12 @@ public abstract class StringHelper {
return 0;
}
};
+
+ public static boolean equals(String s1, String s2) {
+ if (s1 == null) {
+ return s2 == null;
+ } else {
+ return s1.equals(s2);
+ }
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/TermContext.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/TermContext.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/TermContext.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/TermContext.java Thu Jul 7 14:18:02 2011
@@ -91,7 +91,7 @@ public final class TermContext {
final Terms terms = fields.terms(field);
if (terms != null) {
final TermsEnum termsEnum = terms.getThreadTermsEnum(); // thread-private don't share!
- if (SeekStatus.FOUND == termsEnum.seek(bytes, cache)) {
+ if (termsEnum.seekExact(bytes, cache)) {
final TermState termState = termsEnum.termState();
perReaderTermState.register(termState, leaves[i].ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
}
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/Version.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/Version.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/Version.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/Version.java Thu Jul 7 14:18:02 2011
@@ -58,6 +58,13 @@ public enum Version {
*/
@Deprecated
LUCENE_33,
+
+ /**
+ * Match settings and bugs in Lucene's 3.4 release.
+ * @deprecated (4.0) Use latest
+ */
+ @Deprecated
+ LUCENE_34,
/** Match settings and bugs in Lucene's 4.0 release.
* <p>
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/Builder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/Builder.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/Builder.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/Builder.java Thu Jul 7 14:18:02 2011
@@ -21,7 +21,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
-import org.apache.lucene.util.fst.FST.INPUT_TYPE;
+import org.apache.lucene.util.fst.FST.INPUT_TYPE; // javadoc
import java.io.IOException;
@@ -136,6 +136,13 @@ public class Builder<T> {
return dedupHash == null ? 0 : fst.nodeCount;
}
+ /** Pass false to disable the array arc optimization
+ * while building the FST. This is necessary if
+ * encoding a single arc may take more than 255 bytes. */
+ public void setAllowArrayArcs(boolean b) {
+ fst.setAllowArrayArcs(b);
+ }
+
private CompiledNode compileNode(UnCompiledNode<T> n) throws IOException {
final int address;
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/BytesRefFSTEnum.java Thu Jul 7 14:18:02 2011
@@ -71,6 +71,21 @@ public final class BytesRefFSTEnum<T> ex
return setResult();
}
+ /** Seeks to exactly this term, returning null if the term
+ * doesn't exist. This is faster than using {@link
+ * #seekFloor} or {@link #seekCeil} because it
+ * short-circuits as soon the match is not found. */
+ public InputOutput<T> seekExact(BytesRef target) throws IOException {
+ this.target = target;
+ targetLength = target.length;
+ if (super.doSeekExact()) {
+ assert upto == 1+target.length;
+ return setResult();
+ } else {
+ return null;
+ }
+ }
+
@Override
protected int getTargetLabel() {
if (upto-1 == target.length) {
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FST.java Thu Jul 7 14:18:02 2011
@@ -104,6 +104,8 @@ public class FST<T> {
// If arc has this label then that arc is final/accepted
public static final int END_LABEL = -1;
+ private boolean allowArrayArcs = true;
+
public final static class Arc<T> {
public int label;
public T output;
@@ -426,7 +428,7 @@ public class FST<T> {
final int sizeNeeded = fixedArrayStart + node.numArcs * maxBytesPerArc;
bytes = ArrayUtil.grow(bytes, sizeNeeded);
if (maxBytesPerArc > 255) {
- throw new IllegalStateException("max arc size is too large (" + maxBytesPerArc + ")");
+ throw new IllegalStateException("max arc size is too large (" + maxBytesPerArc + "); disable array arcs by calling Builder.setAllowArrayArcs(false)");
}
bytes[fixedArrayStart-1] = (byte) maxBytesPerArc;
@@ -702,6 +704,12 @@ public class FST<T> {
if (labelToMatch == END_LABEL) {
if (follow.isFinal()) {
+ if (follow.target <= 0) {
+ arc.flags = BIT_LAST_ARC;
+ } else {
+ arc.flags = 0;
+ arc.nextArc = follow.target;
+ }
arc.output = follow.nextFinalOutput;
arc.label = END_LABEL;
return arc;
@@ -795,6 +803,10 @@ public class FST<T> {
public int getArcWithOutputCount() {
return arcWithOutputCount;
}
+
+ public void setAllowArrayArcs(boolean v) {
+ allowArrayArcs = v;
+ }
/**
* Nodes will be expanded if their depth (distance from the root node) is
@@ -812,8 +824,9 @@ public class FST<T> {
* @see Builder.UnCompiledNode#depth
*/
private boolean shouldExpand(UnCompiledNode<T> node) {
- return (node.depth <= FIXED_ARRAY_SHALLOW_DISTANCE && node.numArcs >= FIXED_ARRAY_NUM_ARCS_SHALLOW) ||
- node.numArcs >= FIXED_ARRAY_NUM_ARCS_DEEP;
+ return allowArrayArcs &&
+ ((node.depth <= FIXED_ARRAY_SHALLOW_DISTANCE && node.numArcs >= FIXED_ARRAY_NUM_ARCS_SHALLOW) ||
+ node.numArcs >= FIXED_ARRAY_NUM_ARCS_DEEP);
}
// Non-static: writes to FST's byte[]
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/FSTEnum.java Thu Jul 7 14:18:02 2011
@@ -73,6 +73,7 @@ abstract class FSTEnum<T> {
final int cmp = getCurrentLabel() - getTargetLabel();
if (cmp < 0) {
// seek forward
+ //System.out.println(" seek fwd");
break;
} else if (cmp > 0) {
// seek backwards -- reset this arc to the first arc
@@ -83,6 +84,7 @@ abstract class FSTEnum<T> {
}
upto++;
}
+ //System.out.println(" fall through upto=" + upto);
}
protected void doNext() throws IOException {
@@ -352,7 +354,7 @@ abstract class FSTEnum<T> {
//System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1));
fst.readNextRealArc(arc);
assert arc.isLast() || fst.readNextArcLabel(arc) > targetLabel;
- assert arc.label < targetLabel;
+ assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel;
pushLast();
return;
}
@@ -410,6 +412,48 @@ abstract class FSTEnum<T> {
}
}
+ /** Seeks to exactly target term. */
+ protected boolean doSeekExact() throws IOException {
+
+ // TODO: possibly caller could/should provide common
+ // prefix length? ie this work may be redundant if
+ // caller is in fact intersecting against its own
+ // automaton
+
+ //System.out.println("FE: seek exact upto=" + upto);
+
+ // Save time by starting at the end of the shared prefix
+ // b/w our current term & the target:
+ rewindPrefix();
+
+ //System.out.println("FE: after rewind upto=" + upto);
+ FST.Arc<T> arc = getArc(upto-1);
+ int targetLabel = getTargetLabel();
+
+ while(true) {
+ //System.out.println(" cycle target=" + (targetLabel == -1 ? "-1" : (char) targetLabel));
+ final FST.Arc<T> nextArc = fst.findTargetArc(targetLabel, arc, getArc(upto));
+ if (nextArc == null) {
+ // short circuit
+ //upto--;
+ //upto = 0;
+ fst.readFirstTargetArc(arc, getArc(upto));
+ //System.out.println(" no match upto=" + upto);
+ return false;
+ }
+ // Match -- recurse:
+ output[upto] = fst.outputs.add(output[upto-1], nextArc.output);
+ if (targetLabel == FST.END_LABEL) {
+ //System.out.println(" return found; upto=" + upto + " output=" + output[upto] + " nextArc=" + nextArc.isLast());
+ return true;
+ }
+ setCurrentLabel(targetLabel);
+ incr();
+ targetLabel = getTargetLabel();
+ arc = nextArc;
+ }
+ }
+
private void incr() {
upto++;
grow();
Modified: lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/java/org/apache/lucene/util/fst/IntsRefFSTEnum.java Thu Jul 7 14:18:02 2011
@@ -71,6 +71,21 @@ public final class IntsRefFSTEnum<T> ext
return setResult();
}
+ /** Seeks to exactly this term, returning null if the term
+ * doesn't exist. This is faster than using {@link
+ * #seekFloor} or {@link #seekCeil} because it
+ * short-circuits as soon the match is not found. */
+ public InputOutput<T> seekExact(IntsRef target) throws IOException {
+ this.target = target;
+ targetLength = target.length;
+ if (super.doSeekExact()) {
+ assert upto == 1+target.length;
+ return setResult();
+ } else {
+ return null;
+ }
+ }
+
@Override
protected int getTargetLabel() {
if (upto-1 == target.length) {
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/fileformats.html?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/build/site/fileformats.html (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/build/site/fileformats.html Thu Jul 7 14:18:02 2011
@@ -351,7 +351,7 @@ document.write("Last Published: " + docu
languages</a>. If these versions are to remain compatible with Apache
Lucene, then a language-independent definition of the Lucene index
format is required. This document thus attempts to provide a
- complete and independent definition of the Apache Lucene 4.0 file
+ complete and independent definition of the Apache Lucene file
formats.
</p>
<p>
@@ -728,6 +728,14 @@ document.write("Last Published: " + docu
that frequently run out of file handles.</td>
</tr>
+
+<tr>
+
+<td><a href="#Compound File">Compound File Entry table</a></td>
+ <td>.cfe</td>
+ <td>The "virtual" compound file's entry table holding all entries in the corresponding .cfs file (Since 3.4)</td>
+
+</tr>
<tr>
@@ -832,10 +840,10 @@ document.write("Last Published: " + docu
</div>
-<a name="N10204"></a><a name="Primitive Types"></a>
+<a name="N10212"></a><a name="Primitive Types"></a>
<h2 class="boxed">Primitive Types</h2>
<div class="section">
-<a name="N10209"></a><a name="Byte"></a>
+<a name="N10217"></a><a name="Byte"></a>
<h3 class="boxed">Byte</h3>
<p>
The most primitive type
@@ -843,7 +851,7 @@ document.write("Last Published: " + docu
other data types are defined as sequences
of bytes, so file formats are byte-order independent.
</p>
-<a name="N10212"></a><a name="UInt32"></a>
+<a name="N10220"></a><a name="UInt32"></a>
<h3 class="boxed">UInt32</h3>
<p>
32-bit unsigned integers are written as four
@@ -853,7 +861,7 @@ document.write("Last Published: " + docu
UInt32 --> <Byte><sup>4</sup>
</p>
-<a name="N10221"></a><a name="Uint64"></a>
+<a name="N1022F"></a><a name="Uint64"></a>
<h3 class="boxed">Uint64</h3>
<p>
64-bit unsigned integers are written as eight
@@ -862,7 +870,7 @@ document.write("Last Published: " + docu
<p>UInt64 --> <Byte><sup>8</sup>
</p>
-<a name="N10230"></a><a name="VInt"></a>
+<a name="N1023E"></a><a name="VInt"></a>
<h3 class="boxed">VInt</h3>
<p>
A variable-length format for positive integers is
@@ -1412,13 +1420,13 @@ document.write("Last Published: " + docu
This provides compression while still being
efficient to decode.
</p>
-<a name="N10515"></a><a name="Chars"></a>
+<a name="N10523"></a><a name="Chars"></a>
<h3 class="boxed">Chars</h3>
<p>
Lucene writes unicode
character sequences as UTF-8 encoded bytes.
</p>
-<a name="N1051E"></a><a name="String"></a>
+<a name="N1052C"></a><a name="String"></a>
<h3 class="boxed">String</h3>
<p>
Lucene writes strings as UTF-8 encoded bytes.
@@ -1431,10 +1439,10 @@ document.write("Last Published: " + docu
</div>
-<a name="N1052B"></a><a name="Compound Types"></a>
+<a name="N10539"></a><a name="Compound Types"></a>
<h2 class="boxed">Compound Types</h2>
<div class="section">
-<a name="N10530"></a><a name="MapStringString"></a>
+<a name="N1053E"></a><a name="MapStringString"></a>
<h3 class="boxed">Map<String,String></h3>
<p>
In a couple places Lucene stores a Map
@@ -1447,13 +1455,13 @@ document.write("Last Published: " + docu
</div>
-<a name="N10540"></a><a name="Per-Index Files"></a>
+<a name="N1054E"></a><a name="Per-Index Files"></a>
<h2 class="boxed">Per-Index Files</h2>
<div class="section">
<p>
The files in this section exist one-per-index.
</p>
-<a name="N10548"></a><a name="Segments File"></a>
+<a name="N10556"></a><a name="Segments File"></a>
<h3 class="boxed">Segments File</h3>
<p>
The active segments in the index are stored in the
@@ -1626,7 +1634,7 @@ document.write("Last Published: " + docu
<p> HasVectors is 1 if this segment stores term vectors,
else it's 0.
</p>
-<a name="N105D3"></a><a name="Lock File"></a>
+<a name="N105E1"></a><a name="Lock File"></a>
<h3 class="boxed">Lock File</h3>
<p>
The write lock, which is stored in the index
@@ -1640,27 +1648,29 @@ document.write("Last Published: " + docu
documents). This lock file ensures that only one
writer is modifying the index at a time.
</p>
-<a name="N105DC"></a><a name="Deletable File"></a>
+<a name="N105EA"></a><a name="Deletable File"></a>
<h3 class="boxed">Deletable File</h3>
<p>
A writer dynamically computes
the files that are deletable, instead, so no file
is written.
</p>
-<a name="N105E5"></a><a name="Compound Files"></a>
+<a name="N105F3"></a><a name="Compound Files"></a>
<h3 class="boxed">Compound Files</h3>
<p>Starting with Lucene 1.4 the compound file format became default. This
is simply a container for all files described in the next section
(except for the .del file).</p>
-<p>Compound (.cfs) --> FileCount, <DataOffset, FileName>
- <sup>FileCount</sup>
- ,
- FileData
+<p>Compound Entry Table (.cfe) --> Version, FileCount, <FileName, DataOffset, DataLength>
<sup>FileCount</sup>
</p>
+<p>Compound (.cfs) --> FileData <sup>FileCount</sup>
+
+</p>
+<p>Version --> Int</p>
<p>FileCount --> VInt</p>
<p>DataOffset --> Long</p>
+<p>DataLength --> Long</p>
<p>FileName --> String</p>
<p>FileData --> raw file data</p>
<p>The raw file data is the data from the individual files named above.</p>
@@ -1674,14 +1684,14 @@ document.write("Last Published: " + docu
</div>
-<a name="N1060D"></a><a name="Per-Segment Files"></a>
+<a name="N10624"></a><a name="Per-Segment Files"></a>
<h2 class="boxed">Per-Segment Files</h2>
<div class="section">
<p>
The remaining files are all per-segment, and are
thus defined by suffix.
</p>
-<a name="N10615"></a><a name="Fields"></a>
+<a name="N1062C"></a><a name="Fields"></a>
<h3 class="boxed">Fields</h3>
<p>
@@ -1891,7 +1901,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N106D0"></a><a name="Term Dictionary"></a>
+<a name="N106E7"></a><a name="Term Dictionary"></a>
<h3 class="boxed">Term Dictionary</h3>
<p>
The term dictionary is represented as two files:
@@ -2083,7 +2093,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N10754"></a><a name="Frequencies"></a>
+<a name="N1076B"></a><a name="Frequencies"></a>
<h3 class="boxed">Frequencies</h3>
<p>
The .frq file contains the lists of documents
@@ -2211,7 +2221,7 @@ document.write("Last Published: " + docu
entry in level-1. In the example has entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a pointer
to entry 31 on level 0.
</p>
-<a name="N107DC"></a><a name="Positions"></a>
+<a name="N107F3"></a><a name="Positions"></a>
<h3 class="boxed">Positions</h3>
<p>
The .prx file contains the lists of positions that
@@ -2281,7 +2291,7 @@ document.write("Last Published: " + docu
Payload. If PayloadLength is not stored, then this Payload has the same
length as the Payload at the previous position.
</p>
-<a name="N10818"></a><a name="Normalization Factors"></a>
+<a name="N1082F"></a><a name="Normalization Factors"></a>
<h3 class="boxed">Normalization Factors</h3>
<p>There's a single .nrm file containing all norms:
</p>
@@ -2361,7 +2371,7 @@ document.write("Last Published: " + docu
</p>
<p>Separate norm files are created (when adequate) for both compound and non compound segments.
</p>
-<a name="N10869"></a><a name="Term Vectors"></a>
+<a name="N10880"></a><a name="Term Vectors"></a>
<h3 class="boxed">Term Vectors</h3>
<p>
Term Vector support is an optional on a field by
@@ -2497,7 +2507,7 @@ document.write("Last Published: " + docu
</li>
</ol>
-<a name="N10905"></a><a name="Deleted Documents"></a>
+<a name="N1091C"></a><a name="Deleted Documents"></a>
<h3 class="boxed">Deleted Documents</h3>
<p>The .del file is
optional, and only exists when a segment contains deletions.
@@ -2561,7 +2571,7 @@ document.write("Last Published: " + docu
</div>
-<a name="N1093F"></a><a name="Limitations"></a>
+<a name="N10956"></a><a name="Limitations"></a>
<h2 class="boxed">Limitations</h2>
<div class="section">
<p>
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-b-l-15-1body-2menu-3menu.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-b-l-15-1body-2menu-3menu.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-b-r-15-1body-2menu-3menu.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-b-r-15-1body-2menu-3menu.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-b-r-5-1header-2tab-selected-3tab-selected.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2searchbox-3searchbox.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-selected-3tab-selected.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-l-5-1header-2tab-unselected-3tab-unselected.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-15-1body-2menu-3menu.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-15-1body-2menu-3menu.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2searchbox-3searchbox.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-selected-3tab-selected.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/skin/images/rc-t-r-5-1header-2tab-unselected-3tab-unselected.png?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/flexscoring/lucene/src/site/build/site/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/build/site/systemrequirements.html?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/build/site/systemrequirements.html (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/build/site/systemrequirements.html Thu Jul 7 14:18:02 2011
@@ -236,14 +236,11 @@ document.write("Last Published: " + docu
Lucene Java has the following minimum requirements:
<ul>
-<li>Java 1.5.x or greater.</li>
+<li>Java 1.6.x or greater.</li>
<li>ANT 1.7.0 or greater.</li>
-<li>JUnit 3.8.2 if you wish to run the unit tests.</li>
-
-<li>CPU, Disk and Memory requirements are based on the many choices made in implementing Lucene (document size, number of documents, and number of hits retrieved to name a few.)
- The benchmarks page has some information related to performance on particular platforms.</li>
+<li>CPU, Disk and Memory requirements are based on the many choices made in implementing Lucene (document size, number of documents, and number of hits retrieved to name a few.)</li>
</ul>
Modified: lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/changes/changes2html.pl Thu Jul 7 14:18:02 2011
@@ -23,7 +23,6 @@
use strict;
use warnings;
-use LWP::Simple;
my $project_info_url = 'https://issues.apache.org/jira/rest/api/2.0.alpha1/project/LUCENE';
my $jira_url_prefix = 'http://issues.apache.org/jira/browse/';
@@ -674,7 +673,9 @@ sub setup_release_dates {
'2.4.0' => '2008-10-06', '2.4.1' => '2009-03-09',
'2.9.0' => '2009-09-23', '2.9.1' => '2009-11-06',
'3.0.0' => '2009-11-25');
- my $project_info_json = get($project_info_url) || die "Unable to download $project_info_url";
+
+ my $project_info_json = get_url_contents($project_info_url);
+
my $project_info = json2perl($project_info_json);
for my $version (@{$project_info->{versions}}) {
if ($version->{releaseDate}) {
@@ -690,6 +691,21 @@ sub setup_release_dates {
return %release_dates;
}
+#
+# returns contents of the passed in url
+#
+sub get_url_contents {
+ my $url = shift;
+ my $tryWget = `wget --no-check-certificate -O - $url`;
+ if ($? eq 0) {
+ return $tryWget;
+ }
+ my $tryCurl = `curl $url`;
+ if ($? eq 0) {
+ return $tryCurl;
+ }
+ die "could not retrieve $url with either wget or curl!";
+}
#
# setup_month_regex
Modified: lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/fileformats.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/fileformats.xml?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/fileformats.xml (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/fileformats.xml Thu Jul 7 14:18:02 2011
@@ -27,7 +27,7 @@
languages</a>. If these versions are to remain compatible with Apache
Lucene, then a language-independent definition of the Lucene index
format is required. This document thus attempts to provide a
- complete and independent definition of the Apache Lucene 4.0 file
+ complete and independent definition of the Apache Lucene file
formats.
</p>
@@ -366,6 +366,11 @@
<td>An optional "virtual" file consisting of all the other index files for systems
that frequently run out of file handles.</td>
</tr>
+ <tr>
+ <td><a href="#Compound File">Compound File Entry table</a></td>
+ <td>.cfe</td>
+ <td>The "virtual" compound file's entry table holding all entries in the corresponding .cfs file (Since 3.4)</td>
+ </tr>
<tr>
<td><a href="#Fields">Fields</a></td>
<td>.fnm</td>
@@ -1129,17 +1134,20 @@
<p>Starting with Lucene 1.4 the compound file format became default. This
is simply a container for all files described in the next section
(except for the .del file).</p>
-
- <p>Compound (.cfs) --> FileCount, <DataOffset, FileName>
- <sup>FileCount</sup>
- ,
- FileData
+ <p>Compound Entry Table (.cfe) --> Version, FileCount, <FileName, DataOffset, DataLength>
<sup>FileCount</sup>
</p>
+ <p>Compound (.cfs) --> FileData <sup>FileCount</sup>
+ </p>
+
+ <p>Version --> Int</p>
+
<p>FileCount --> VInt</p>
<p>DataOffset --> Long</p>
+
+ <p>DataLength --> Long</p>
<p>FileName --> String</p>
Modified: lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/systemrequirements.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/systemrequirements.xml?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/systemrequirements.xml (original)
+++ lucene/dev/branches/flexscoring/lucene/src/site/src/documentation/content/xdocs/systemrequirements.xml Thu Jul 7 14:18:02 2011
@@ -13,11 +13,9 @@
<p>
Lucene Java has the following minimum requirements:
<ul>
- <li>Java 1.5.x or greater.</li>
+ <li>Java 1.6.x or greater.</li>
<li>ANT 1.7.0 or greater.</li>
- <li>JUnit 3.8.2 if you wish to run the unit tests.</li>
- <li>CPU, Disk and Memory requirements are based on the many choices made in implementing Lucene (document size, number of documents, and number of hits retrieved to name a few.)
- The benchmarks page has some information related to performance on particular platforms.</li>
+ <li>CPU, Disk and Memory requirements are based on the many choices made in implementing Lucene (document size, number of documents, and number of hits retrieved to name a few.)</li>
</ul>
</p>
<p>Contrib modules may have other requirements, refer to their documentation and build files for information.</p>
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Thu Jul 7 14:18:02 2011
@@ -264,17 +264,64 @@ public abstract class BaseTokenStreamTes
TokenStream ts = a.reusableTokenStream("dummy", new StringReader(text));
assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class));
CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class);
+ OffsetAttribute offsetAtt = ts.hasAttribute(OffsetAttribute.class) ? ts.getAttribute(OffsetAttribute.class) : null;
+ PositionIncrementAttribute posIncAtt = ts.hasAttribute(PositionIncrementAttribute.class) ? ts.getAttribute(PositionIncrementAttribute.class) : null;
+ TypeAttribute typeAtt = ts.hasAttribute(TypeAttribute.class) ? ts.getAttribute(TypeAttribute.class) : null;
List<String> tokens = new ArrayList<String>();
+ List<String> types = new ArrayList<String>();
+ List<Integer> positions = new ArrayList<Integer>();
+ List<Integer> startOffsets = new ArrayList<Integer>();
+ List<Integer> endOffsets = new ArrayList<Integer>();
ts.reset();
while (ts.incrementToken()) {
tokens.add(termAtt.toString());
- // TODO: we could collect offsets etc here for better checking that reset() really works.
+ if (typeAtt != null) types.add(typeAtt.type());
+ if (posIncAtt != null) positions.add(posIncAtt.getPositionIncrement());
+ if (offsetAtt != null) {
+ startOffsets.add(offsetAtt.startOffset());
+ endOffsets.add(offsetAtt.endOffset());
+ }
}
ts.end();
ts.close();
// verify reusing is "reproducable" and also get the normal tokenstream sanity checks
- if (!tokens.isEmpty())
- assertAnalyzesToReuse(a, text, tokens.toArray(new String[tokens.size()]));
+ if (!tokens.isEmpty()) {
+ if (typeAtt != null && posIncAtt != null && offsetAtt != null) {
+ // offset + pos + type
+ assertAnalyzesToReuse(a, text,
+ tokens.toArray(new String[tokens.size()]),
+ toIntArray(startOffsets),
+ toIntArray(endOffsets),
+ types.toArray(new String[types.size()]),
+ toIntArray(positions));
+ } else if (posIncAtt != null && offsetAtt != null) {
+ // offset + pos
+ assertAnalyzesToReuse(a, text,
+ tokens.toArray(new String[tokens.size()]),
+ toIntArray(startOffsets),
+ toIntArray(endOffsets),
+ toIntArray(positions));
+ } else if (offsetAtt != null) {
+ // offset
+ assertAnalyzesToReuse(a, text,
+ tokens.toArray(new String[tokens.size()]),
+ toIntArray(startOffsets),
+ toIntArray(endOffsets));
+ } else {
+ // terms only
+ assertAnalyzesToReuse(a, text,
+ tokens.toArray(new String[tokens.size()]));
+ }
+ }
+ }
+ }
+
+ static int[] toIntArray(List<Integer> list) {
+ int ret[] = new int[list.size()];
+ int offset = 0;
+ for (Integer i : list) {
+ ret[offset++] = i;
}
+ return ret;
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java Thu Jul 7 14:18:02 2011
@@ -22,7 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Random;
-import java.util.Set;
+import java.util.Map;
import org.apache.lucene.util._TestUtil;
@@ -56,12 +56,12 @@ public class MockRandomMergePolicy exten
@Override
public MergeSpecification findMergesForOptimize(
- SegmentInfos segmentInfos, int maxSegmentCount, Set<SegmentInfo> segmentsToOptimize)
+ SegmentInfos segmentInfos, int maxSegmentCount, Map<SegmentInfo,Boolean> segmentsToOptimize)
throws CorruptIndexException, IOException {
final List<SegmentInfo> eligibleSegments = new ArrayList<SegmentInfo>();
for(SegmentInfo info : segmentInfos) {
- if (segmentsToOptimize.contains(info)) {
+ if (segmentsToOptimize.containsKey(info)) {
eligibleSegments.add(info);
}
}
@@ -85,7 +85,7 @@ public class MockRandomMergePolicy exten
if (mergeSpec != null) {
for(OneMerge merge : mergeSpec.merges) {
for(SegmentInfo info : merge.segments) {
- assert segmentsToOptimize.contains(info);
+ assert segmentsToOptimize.containsKey(info);
}
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Thu Jul 7 14:18:02 2011
@@ -114,7 +114,7 @@ public class RandomIndexWriter implement
private void switchDoDocValues() {
// randomly enable / disable docValues
- doDocValues = r.nextInt(10) != 0;
+ doDocValues = LuceneTestCase.rarely(r);
}
/**
@@ -133,22 +133,22 @@ public class RandomIndexWriter implement
// addDocuments? Would be better testing.
w.addDocuments(new Iterable<Document>() {
- // @Override -- not until Java 1.6
+ @Override
public Iterator<Document> iterator() {
return new Iterator<Document>() {
boolean done;
- // @Override -- not until Java 1.6
+ @Override
public boolean hasNext() {
return !done;
}
- // @Override -- not until Java 1.6
+ @Override
public void remove() {
throw new UnsupportedOperationException();
}
- // @Override -- not until Java 1.6
+ @Override
public Document next() {
if (done) {
throw new IllegalStateException();
@@ -200,9 +200,21 @@ public class RandomIndexWriter implement
case FLOAT_64:
docValuesField.setFloat(random.nextDouble());
break;
- case INTS:
+ case VAR_INTS:
+ docValuesField.setInt(random.nextLong());
+ break;
+ case FIXED_INTS_16:
+ docValuesField.setInt(random.nextInt(Short.MAX_VALUE));
+ break;
+ case FIXED_INTS_32:
docValuesField.setInt(random.nextInt());
break;
+ case FIXED_INTS_64:
+ docValuesField.setInt(random.nextLong());
+ break;
+ case FIXED_INTS_8:
+ docValuesField.setInt(random.nextInt(128));
+ break;
default:
throw new IllegalArgumentException("no such type: " + type);
}
@@ -247,22 +259,22 @@ public class RandomIndexWriter implement
if (r.nextInt(5) == 3) {
w.updateDocuments(t, new Iterable<Document>() {
- // @Override -- not until Java 1.6
+ @Override
public Iterator<Document> iterator() {
return new Iterator<Document>() {
boolean done;
- // @Override -- not until Java 1.6
+ @Override
public boolean hasNext() {
return !done;
}
- // @Override -- not until Java 1.6
+ @Override
public void remove() {
throw new UnsupportedOperationException();
}
- // @Override -- not until Java 1.6
+ @Override
public Document next() {
if (done) {
throw new IllegalStateException();
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java Thu Jul 7 14:18:02 2011
@@ -106,6 +106,7 @@ public class CheckHits {
Assert.assertEquals("Wrap Reader " + i + ": " +
query.toString(defaultFieldName),
correct, actual);
+ FieldCache.DEFAULT.purge(s.getIndexReader()); // our wrapping can create insanity otherwise
s.close();
}
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java Thu Jul 7 14:18:02 2011
@@ -114,10 +114,13 @@ public class QueryUtils {
if (wrap) {
IndexSearcher wrapped;
check(random, q1, wrapped = wrapUnderlyingReader(random, s, -1), false);
+ FieldCache.DEFAULT.purge(wrapped.getIndexReader()); // // our wrapping can create insanity otherwise
wrapped.close();
check(random, q1, wrapped = wrapUnderlyingReader(random, s, 0), false);
+ FieldCache.DEFAULT.purge(wrapped.getIndexReader()); // // our wrapping can create insanity otherwise
wrapped.close();
check(random, q1, wrapped = wrapUnderlyingReader(random, s, +1), false);
+ FieldCache.DEFAULT.purge(wrapped.getIndexReader()); // // our wrapping can create insanity otherwise
wrapped.close();
}
checkExplanations(q1,s);
@@ -148,23 +151,35 @@ public class QueryUtils {
// we can't put deleted docs before the nested reader, because
// it will throw off the docIds
IndexReader[] readers = new IndexReader[] {
- edge < 0 ? r : IndexReader.open(makeEmptyIndex(random, 0), true),
- IndexReader.open(makeEmptyIndex(random, 0), true),
- new MultiReader(IndexReader.open(makeEmptyIndex(random, edge < 0 ? 4 : 0), true),
- IndexReader.open(makeEmptyIndex(random, 0), true),
- 0 == edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true)),
- IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 7), true),
- IndexReader.open(makeEmptyIndex(random, 0), true),
- new MultiReader(IndexReader.open(makeEmptyIndex(random, 0 < edge ? 0 : 5), true),
- IndexReader.open(makeEmptyIndex(random, 0), true),
- 0 < edge ? r : IndexReader.open(makeEmptyIndex(random, 0), true))
+ edge < 0 ? r : emptyReaders[0],
+ emptyReaders[0],
+ new MultiReader(edge < 0 ? emptyReaders[4] : emptyReaders[0],
+ emptyReaders[0],
+ 0 == edge ? r : emptyReaders[0]),
+ 0 < edge ? emptyReaders[0] : emptyReaders[7],
+ emptyReaders[0],
+ new MultiReader(0 < edge ? emptyReaders[0] : emptyReaders[5],
+ emptyReaders[0],
+ 0 < edge ? r : emptyReaders[0])
};
IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
out.setSimilarityProvider(s.getSimilarityProvider());
return out;
}
+
+ static final IndexReader[] emptyReaders = new IndexReader[8];
+ static {
+ try {
+ emptyReaders[0] = makeEmptyIndex(new Random(0), 0);
+ emptyReaders[4] = makeEmptyIndex(new Random(0), 4);
+ emptyReaders[5] = makeEmptyIndex(new Random(0), 5);
+ emptyReaders[7] = makeEmptyIndex(new Random(0), 7);
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
- private static Directory makeEmptyIndex(Random random, final int numDeletedDocs)
+ private static IndexReader makeEmptyIndex(Random random, final int numDeletedDocs)
throws IOException {
Directory d = new MockDirectoryWrapper(random, new RAMDirectory());
IndexWriter w = new IndexWriter(d, new IndexWriterConfig(
@@ -188,8 +203,7 @@ public class QueryUtils {
IndexReader r = IndexReader.open(d, true);
Assert.assertEquals("reader has wrong number of deleted docs",
numDeletedDocs, r.numDeletedDocs());
- r.close();
- return d;
+ return r;
}
/** alternate scorer skipTo(),skipTo(),next(),next(),skipTo(),skipTo(), etc
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java Thu Jul 7 14:18:02 2011
@@ -388,7 +388,7 @@ public class MockDirectoryWrapper extend
}
}
- private void addFileHandle(Closeable c, String name, boolean input) {
+ synchronized void addFileHandle(Closeable c, String name, boolean input) {
Integer v = openFiles.get(name);
if (v != null) {
v = Integer.valueOf(v.intValue()+1);
@@ -416,6 +416,18 @@ public class MockDirectoryWrapper extend
addFileHandle(ii, name, true);
return ii;
}
+
+ @Override
+ public synchronized CompoundFileDirectory openCompoundInput(String name, int bufferSize) throws IOException {
+ maybeYield();
+ return new MockCompoundFileDirectoryWrapper(name, this, delegate.openCompoundInput(name, bufferSize), false);
+ }
+
+ @Override
+ public CompoundFileDirectory createCompoundOutput(String name) throws IOException {
+ maybeYield();
+ return new MockCompoundFileDirectoryWrapper(name, this, delegate.createCompoundOutput(name), true);
+ }
/** Provided for testing purposes. Use sizeInBytes() instead. */
public synchronized final long getRecomputedSizeInBytes() throws IOException {
@@ -481,7 +493,7 @@ public class MockDirectoryWrapper extend
delegate.close();
}
- private synchronized void removeOpenFile(Closeable c, String name) {
+ synchronized void removeOpenFile(Closeable c, String name) {
Integer v = openFiles.get(name);
// Could be null when crash() was called
if (v != null) {
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java Thu Jul 7 14:18:02 2011
@@ -137,4 +137,14 @@ public class MockIndexInputWrapper exten
public Map<String,String> readStringStringMap() throws IOException {
return delegate.readStringStringMap();
}
+
+ @Override
+ public int readVInt() throws IOException {
+ return delegate.readVInt();
+ }
+
+ @Override
+ public long readVLong() throws IOException {
+ return delegate.readVLong();
+ }
}
Modified: lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1143840&r1=1143839&r2=1143840&view=diff
==============================================================================
--- lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/flexscoring/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Thu Jul 7 14:18:02 2011
@@ -52,6 +52,7 @@ import org.apache.lucene.index.codecs.pr
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.index.codecs.memory.MemoryCodec;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.FieldCache.CacheEntry;
@@ -632,6 +633,11 @@ public abstract class LuceneTestCase ext
for (Thread t : Thread.getAllStackTraces().keySet()) {
rogueThreads.put(t, true);
}
+
+ if (TEST_ITER > 1) {
+ System.out.println("WARNING: you are using -Dtests.iter=n where n > 1, not all tests support this option.");
+ System.out.println("Some may crash or fail: this is not a bug.");
+ }
}
/**
@@ -712,8 +718,12 @@ public abstract class LuceneTestCase ext
throw e;
}
+ if (insanity.length != 0) {
+ reportAdditionalFailureInfo();
+ }
+
assertEquals(msg + ": Insane FieldCache usage(s) found",
- 0, insanity.length);
+ 0, insanity.length);
insanity = null;
} finally {
@@ -947,6 +957,7 @@ public abstract class LuceneTestCase ext
tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20));
tmp.setUseCompoundFile(r.nextBoolean());
tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8);
+ tmp.setReclaimDeletesWeight(r.nextDouble()*4);
return tmp;
}
@@ -1102,11 +1113,17 @@ public abstract class LuceneTestCase ext
/** Returns a new field instance, using the specified random.
* See {@link #newField(String, String, Field.Store, Field.Index, Field.TermVector)} for more information */
public static Field newField(Random random, String name, String value, Store store, Index index, TermVector tv) {
+
if (usually(random)) {
// most of the time, don't modify the params
return new Field(name, value, store, index, tv);
}
+ if (random.nextBoolean()) {
+ // tickle any code still relying on field names being interned:
+ name = new String(name);
+ }
+
if (!index.isIndexed())
return new Field(name, value, store, index, tv);
@@ -1451,6 +1468,7 @@ public abstract class LuceneTestCase ext
register(new PreFlexCodec());
register(new PulsingCodec(1));
register(new SimpleTextCodec());
+ register(new MemoryCodec());
Collections.shuffle(knownCodecs, random);
}