You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/08/25 12:06:16 UTC
svn commit: r1377246 [5/7] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/idea/lucene/analysis/morfologik/
dev-tools/idea/lucene/analysis/phonetic/ dev-tools/maven/
dev-tools/scripts/ lucene/ lucene/analysis/ lucene/analysis/common/ luce...
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java Sat Aug 25 10:06:07 2012
@@ -17,12 +17,12 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
-import org.apache.lucene.store.DataInput;
-import org.apache.lucene.util.RamUsageEstimator;
-
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.util.RamUsageEstimator;
+
/**
* Space optimized random access capable array of values with a fixed number of
* bits/value. Values are packed contiguously.
@@ -146,12 +146,12 @@ class Packed64 extends PackedInts.Mutabl
assert off + len <= arr.length;
final int originalIndex = index;
- final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
+ final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
// go to the next block where the value does not span across two blocks
- final int offsetInBlocks = index % op.values();
+ final int offsetInBlocks = index % decoder.valueCount();
if (offsetInBlocks != 0) {
- for (int i = offsetInBlocks; i < op.values() && len > 0; ++i) {
+ for (int i = offsetInBlocks; i < decoder.valueCount() && len > 0; ++i) {
arr[off++] = get(index++);
--len;
}
@@ -161,12 +161,12 @@ class Packed64 extends PackedInts.Mutabl
}
// bulk get
- assert index % op.values() == 0;
+ assert index % decoder.valueCount() == 0;
int blockIndex = (int) ((long) index * bitsPerValue) >>> BLOCK_BITS;
assert (((long)index * bitsPerValue) & MOD_MASK) == 0;
- final int iterations = len / op.values();
- op.get(blocks, blockIndex, arr, off, iterations);
- final int gotValues = iterations * op.values();
+ final int iterations = len / decoder.valueCount();
+ decoder.decode(blocks, blockIndex, arr, off, iterations);
+ final int gotValues = iterations * decoder.valueCount();
index += gotValues;
len -= gotValues;
assert len >= 0;
@@ -210,12 +210,12 @@ class Packed64 extends PackedInts.Mutabl
assert off + len <= arr.length;
final int originalIndex = index;
- final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
+ final PackedInts.Encoder encoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
// go to the next block where the value does not span across two blocks
- final int offsetInBlocks = index % op.values();
+ final int offsetInBlocks = index % encoder.valueCount();
if (offsetInBlocks != 0) {
- for (int i = offsetInBlocks; i < op.values() && len > 0; ++i) {
+ for (int i = offsetInBlocks; i < encoder.valueCount() && len > 0; ++i) {
set(index++, arr[off++]);
--len;
}
@@ -224,13 +224,13 @@ class Packed64 extends PackedInts.Mutabl
}
}
- // bulk get
- assert index % op.values() == 0;
+ // bulk set
+ assert index % encoder.valueCount() == 0;
int blockIndex = (int) ((long) index * bitsPerValue) >>> BLOCK_BITS;
assert (((long)index * bitsPerValue) & MOD_MASK) == 0;
- final int iterations = len / op.values();
- op.set(blocks, blockIndex, arr, off, iterations);
- final int setValues = iterations * op.values();
+ final int iterations = len / encoder.valueCount();
+ encoder.encode(arr, off, blocks, blockIndex, iterations);
+ final int setValues = iterations * encoder.valueCount();
index += setValues;
len -= setValues;
assert len >= 0;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java Sat Aug 25 10:06:07 2012
@@ -86,12 +86,12 @@ abstract class Packed64SingleBlock exten
// bulk get
assert index % valuesPerBlock == 0;
- final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
- assert op.blocks() == 1;
- assert op.values() == valuesPerBlock;
+ final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
+ assert decoder.blockCount() == 1;
+ assert decoder.valueCount() == valuesPerBlock;
final int blockIndex = index / valuesPerBlock;
final int nblocks = (index + len) / valuesPerBlock - blockIndex;
- op.get(blocks, blockIndex, arr, off, nblocks);
+ decoder.decode(blocks, blockIndex, arr, off, nblocks);
final int diff = nblocks * valuesPerBlock;
index += diff; len -= diff;
@@ -131,11 +131,11 @@ abstract class Packed64SingleBlock exten
// bulk set
assert index % valuesPerBlock == 0;
final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
- assert op.blocks() == 1;
- assert op.values() == valuesPerBlock;
+ assert op.blockCount() == 1;
+ assert op.valueCount() == valuesPerBlock;
final int blockIndex = index / valuesPerBlock;
final int nblocks = (index + len) / valuesPerBlock - blockIndex;
- op.set(blocks, blockIndex, arr, off, nblocks);
+ op.encode(arr, off, blocks, blockIndex, nblocks);
final int diff = nblocks * valuesPerBlock;
index += diff; len -= diff;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Sat Aug 25 10:06:07 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.util.packed;
*/
import java.io.Closeable;
+import java.io.IOException;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
@@ -25,8 +26,6 @@ import org.apache.lucene.store.DataOutpu
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.LongsRef;
-import java.io.IOException;
-
/**
* Simplistic compression for array of unsigned long values.
* Each value is >= 0 and <= a specified maximum value. The
@@ -66,6 +65,14 @@ public class PackedInts {
public final static int VERSION_START = 0;
public final static int VERSION_CURRENT = VERSION_START;
+ private static void checkVersion(int version) {
+ if (version < VERSION_START) {
+ throw new IllegalArgumentException("Version is too old, should be at least " + VERSION_START + " (got " + version + ")");
+ } else if (version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version is too new, should be at most " + VERSION_CURRENT + " (got " + version + ")");
+ }
+ }
+
/**
* A format to write packed ints.
*
@@ -242,6 +249,146 @@ public class PackedInts {
}
/**
+ * A decoder for packed integers.
+ */
+ public static interface Decoder {
+
+ /**
+ * The minimum number of long blocks to decode in a single call.
+ */
+ int blockCount();
+
+ /**
+ * The number of values that can be stored in <code>blockCount()</code> long
+ * blocks.
+ */
+ int valueCount();
+
+ /**
+ * Read <code>iterations * blockCount()</code> blocks from <code>blocks</code>,
+ * decode them and write <code>iterations * valueCount()</code> values into
+ * <code>values</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start reading blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start writing values
+ * @param iterations controls how much data to decode
+ */
+ void decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations);
+
+ /**
+ * Read <code>8 * iterations * blockCount()</code> blocks from <code>blocks</code>,
+ * decode them and write <code>iterations * valueCount()</code> values into
+ * <code>values</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start reading blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start writing values
+ * @param iterations controls how much data to decode
+ */
+ void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations);
+
+ /**
+ * Read <code>iterations * blockCount()</code> blocks from <code>blocks</code>,
+ * decode them and write <code>iterations * valueCount()</code> values into
+ * <code>values</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start reading blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start writing values
+ * @param iterations controls how much data to decode
+ */
+ void decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations);
+
+ /**
+ * Read <code>8 * iterations * blockCount()</code> blocks from <code>blocks</code>,
+ * decode them and write <code>iterations * valueCount()</code> values into
+ * <code>values</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start reading blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start writing values
+ * @param iterations controls how much data to decode
+ */
+ void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations);
+
+ }
+
+ /**
+ * An encoder for packed integers.
+ */
+ public static interface Encoder {
+
+ /**
+ * The minimum number of long blocks to encode in a single call.
+ */
+ int blockCount();
+
+ /**
+ * The number of values that can be stored in <code>blockCount()</code> long
+ * blocks.
+ */
+ int valueCount();
+
+ /**
+ * Read <code>iterations * valueCount()</code> values from <code>values</code>,
+ * encode them and write <code>iterations * blockCount()</code> blocks into
+ * <code>blocks</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start writing blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start reading values
+ * @param iterations controls how much data to encode
+ */
+ void encode(long[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations);
+
+ /**
+ * Read <code>iterations * valueCount()</code> values from <code>values</code>,
+ * encode them and write <code>8 * iterations * blockCount()</code> blocks into
+ * <code>blocks</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start writing blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start reading values
+ * @param iterations controls how much data to encode
+ */
+ void encode(long[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations);
+
+ /**
+ * Read <code>iterations * valueCount()</code> values from <code>values</code>,
+ * encode them and write <code>iterations * blockCount()</code> blocks into
+ * <code>blocks</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start writing blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start reading values
+ * @param iterations controls how much data to encode
+ */
+ void encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations);
+
+ /**
+ * Read <code>iterations * valueCount()</code> values from <code>values</code>,
+ * encode them and write <code>8 * iterations * blockCount()</code> blocks into
+ * <code>blocks</code>.
+ *
+ * @param blocks the long blocks that hold packed integer values
+ * @param blocksOffset the offset where to start writing blocks
+ * @param values the values buffer
+ * @param valuesOffset the offset where to start reading values
+ * @param iterations controls how much data to encode
+ */
+ void encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations);
+
+ }
+
+ /**
* A read-only random access array of positive integers.
* @lucene.internal
*/
@@ -490,8 +637,7 @@ public class PackedInts {
protected final int valueCount;
protected final int bitsPerValue;
- protected Writer(DataOutput out, int valueCount, int bitsPerValue)
- throws IOException {
+ protected Writer(DataOutput out, int valueCount, int bitsPerValue) {
assert bitsPerValue <= 64;
assert valueCount >= 0 || valueCount == -1;
this.out = out;
@@ -529,6 +675,32 @@ public class PackedInts {
}
/**
+ * Get a {@link Decoder}.
+ *
+ * @param format the format used to store packed ints
+ * @param version the compatibility version
+ * @param bitsPerValue the number of bits per value
+ * @return a decoder
+ */
+ public static Decoder getDecoder(Format format, int version, int bitsPerValue) {
+ checkVersion(version);
+ return BulkOperation.of(format, bitsPerValue);
+ }
+
+ /**
+ * Get an {@link Encoder}.
+ *
+ * @param format the format used to store packed ints
+ * @param version the compatibility version
+ * @param bitsPerValue the number of bits per value
+ * @return an encoder
+ */
+ public static Encoder getEncoder(Format format, int version, int bitsPerValue) {
+ checkVersion(version);
+ return BulkOperation.of(format, bitsPerValue);
+ }
+
+ /**
* Expert: Restore a {@link Reader} from a stream without reading metadata at
* the beginning of the stream. This method is useful to restore data from
* streams which have been created using
@@ -546,6 +718,7 @@ public class PackedInts {
*/
public static Reader getReaderNoHeader(DataInput in, Format format, int version,
int valueCount, int bitsPerValue) throws IOException {
+ checkVersion(version);
switch (format) {
case PACKED_SINGLE_BLOCK:
return Packed64SingleBlock.create(in, valueCount, bitsPerValue);
@@ -612,7 +785,8 @@ public class PackedInts {
* @lucene.internal
*/
public static ReaderIterator getReaderIteratorNoHeader(DataInput in, Format format, int version,
- int valueCount, int bitsPerValue, int mem) throws IOException {
+ int valueCount, int bitsPerValue, int mem) {
+ checkVersion(version);
return new PackedReaderIterator(format, valueCount, bitsPerValue, in, mem);
}
@@ -652,7 +826,8 @@ public class PackedInts {
* @lucene.internal
*/
public static Reader getDirectReaderNoHeader(IndexInput in, Format format,
- int version, int valueCount, int bitsPerValue) throws IOException {
+ int version, int valueCount, int bitsPerValue) {
+ checkVersion(version);
switch (format) {
case PACKED:
return new DirectPackedReader(bitsPerValue, valueCount, in);
@@ -784,7 +959,7 @@ public class PackedInts {
* @lucene.internal
*/
public static Writer getWriterNoHeader(
- DataOutput out, Format format, int valueCount, int bitsPerValue, int mem) throws IOException {
+ DataOutput out, Format format, int valueCount, int bitsPerValue, int mem) {
return new PackedWriter(format, out, valueCount, bitsPerValue, mem);
}
@@ -894,4 +1069,4 @@ public class PackedInts {
}
}
-}
+}
\ No newline at end of file
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java Sat Aug 25 10:06:07 2012
@@ -38,10 +38,10 @@ final class PackedReaderIterator extends
bulkOperation = BulkOperation.of(format, bitsPerValue);
iterations = bulkOperation.computeIterations(valueCount, mem);
assert iterations > 0;
- nextBlocks = new long[iterations * bulkOperation.blocks()];
- nextValues = new LongsRef(new long[iterations * bulkOperation.values()], 0, 0);
- assert iterations * bulkOperation.values() == nextValues.longs.length;
- assert iterations * bulkOperation.blocks() == nextBlocks.length;
+ nextBlocks = new long[iterations * bulkOperation.blockCount()];
+ nextValues = new LongsRef(new long[iterations * bulkOperation.valueCount()], 0, 0);
+ assert iterations * bulkOperation.valueCount() == nextValues.longs.length;
+ assert iterations * bulkOperation.blockCount() == nextBlocks.length;
nextValues.offset = nextValues.longs.length;
position = -1;
}
@@ -70,7 +70,7 @@ final class PackedReaderIterator extends
nextBlocks[i] = 0L;
}
- bulkOperation.get(nextBlocks, 0, nextValues.longs, 0, iterations);
+ bulkOperation.decode(nextBlocks, 0, nextValues.longs, 0, iterations);
nextValues.offset = 0;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java Sat Aug 25 10:06:07 2012
@@ -21,6 +21,7 @@ import org.apache.lucene.store.DataOutpu
import java.io.EOFException;
import java.io.IOException;
+import java.util.Arrays;
// Packs high order byte first, to match
// IndexOutput.writeInt/Long/Short byte order
@@ -29,21 +30,20 @@ final class PackedWriter extends PackedI
boolean finished;
final PackedInts.Format format;
- final BulkOperation bulkOperation;
+ final BulkOperation encoder;
final long[] nextBlocks;
final long[] nextValues;
final int iterations;
int off;
int written;
- PackedWriter(PackedInts.Format format, DataOutput out, int valueCount, int bitsPerValue, int mem)
- throws IOException {
+ PackedWriter(PackedInts.Format format, DataOutput out, int valueCount, int bitsPerValue, int mem) {
super(out, valueCount, bitsPerValue);
this.format = format;
- bulkOperation = BulkOperation.of(format, bitsPerValue);
- iterations = bulkOperation.computeIterations(valueCount, mem);
- nextBlocks = new long[iterations * bulkOperation.blocks()];
- nextValues = new long[iterations * bulkOperation.values()];
+ encoder = BulkOperation.of(format, bitsPerValue);
+ iterations = encoder.computeIterations(valueCount, mem);
+ nextBlocks = new long[iterations * encoder.blockCount()];
+ nextValues = new long[iterations * encoder.valueCount()];
off = 0;
written = 0;
finished = false;
@@ -63,8 +63,7 @@ final class PackedWriter extends PackedI
}
nextValues[off++] = v;
if (off == nextValues.length) {
- flush(nextValues.length);
- off = 0;
+ flush();
}
++written;
}
@@ -77,16 +76,17 @@ final class PackedWriter extends PackedI
add(0L);
}
}
- flush(off);
+ flush();
finished = true;
}
- private void flush(int nvalues) throws IOException {
- bulkOperation.set(nextBlocks, 0, nextValues, 0, iterations);
- final int blocks = format.nblocks(bitsPerValue, nvalues);
+ private void flush() throws IOException {
+ encoder.encode(nextValues, 0, nextBlocks, 0, iterations);
+ final int blocks = format.nblocks(bitsPerValue, off);
for (int i = 0; i < blocks; ++i) {
out.writeLong(nextBlocks[i]);
}
+ Arrays.fill(nextValues, 0L);
off = 0;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py Sat Aug 25 10:06:07 2012
@@ -42,22 +42,35 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
-import java.util.EnumMap;
-
-/**
- * Efficient sequential read/write of packed integers.
- */
-abstract class BulkOperation {
+"""
- static final EnumMap<PackedInts.Format, BulkOperation[]> BULK_OPERATIONS = new EnumMap<PackedInts.Format, BulkOperation[]>(PackedInts.Format.class);
+FOOTER="""
- public static BulkOperation of(PackedInts.Format format, int bitsPerValue) {
- assert bitsPerValue > 0 && bitsPerValue <= 64;
- BulkOperation[] ops = BULK_OPERATIONS.get(format);
- if (ops == null || ops[bitsPerValue] == null) {
- throw new IllegalArgumentException("format: " + format + ", bitsPerValue: " + bitsPerValue);
+ private static long[] toLongArray(int[] ints, int offset, int length) {
+ long[] arr = new long[length];
+ for (int i = 0; i < length; ++i) {
+ arr[i] = ints[offset + i];
}
- return ops[bitsPerValue];
+ return arr;
+ }
+
+ @Override
+ public void encode(int[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {
+ encode(toLongArray(values, valuesOffset, iterations * valueCount()), 0, blocks, blocksOffset, iterations);
+ }
+
+ @Override
+ public void encode(long[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations) {
+ final long[] longBLocks = new long[blockCount() * iterations];
+ encode(values, valuesOffset, longBLocks, 0, iterations);
+ ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer().put(longBLocks);
+ }
+
+ @Override
+ public void encode(int[] values, int valuesOffset, byte[] blocks, int blocksOffset, int iterations) {
+ final long[] longBLocks = new long[blockCount() * iterations];
+ encode(values, valuesOffset, longBLocks, 0, iterations);
+ ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer().put(longBLocks);
}
/**
@@ -67,7 +80,7 @@ abstract class BulkOperation {
* - 16 bits per value -> b=1, v=4
* - 24 bits per value -> b=3, v=8
* - 50 bits per value -> b=25, v=32
- * - 63 bits per value -> b=63, v = 64
+ * - 63 bits per value -> b=63, v=64
* - ...
*
* A bulk read consists in copying <code>iterations*v</code> values that are
@@ -79,87 +92,165 @@ abstract class BulkOperation {
* <code>ramBudget / (8 * (b + v))</code> (since a long is 8 bytes).
*/
public final int computeIterations(int valueCount, int ramBudget) {
- final int iterations = (ramBudget >>> 3) / (blocks() + values());
+ final int iterations = (ramBudget >>> 3) / (blockCount() + valueCount());
if (iterations == 0) {
// at least 1
return 1;
- } else if ((iterations - 1) * blocks() >= valueCount) {
+ } else if ((iterations - 1) * blockCount() >= valueCount) {
// don't allocate for more than the size of the reader
- return (int) Math.ceil((double) valueCount / values());
+ return (int) Math.ceil((double) valueCount / valueCount());
} else {
return iterations;
}
}
-
- /**
- * The minimum number of blocks required to perform a bulk get/set.
- */
- public abstract int blocks();
-
- /**
- * The number of values that can be stored in <code>blocks()</code> blocks.
- */
- public abstract int values();
-
- /**
- * Get <code>n * values()</code> values from <code>n * blocks()</code> blocks.
- */
- public abstract void get(long[] blocks, int blockIndex, long[] values, int valuesIndex, int iterations);
-
- /**
- * Set <code>n * values()</code> values into <code>n * blocks()</code> blocks.
- */
- public abstract void set(long[] blocks, int blockIndex, long[] values, int valuesIndex, int iterations);
-
+}
"""
-FOOTER = "}"
+def casts(typ):
+ cast_start = "(%s) (" %typ
+ cast_end = ")"
+ if typ == "long":
+ cast_start = ""
+ cast_end = ""
+ return cast_start, cast_end
+
+def hexNoLSuffix(n):
+ # On 32 bit Python values > (1 << 31)-1 will have L appended by hex function:
+ s = hex(n)
+ if s.endswith('L'):
+ s = s[:-1]
+ return s
+
+def masks(bits):
+ if bits == 64:
+ return "", ""
+ return "(", " & %sL)" %(hexNoLSuffix((1 << bits) - 1))
+
+def get_type(bits):
+ if bits == 8:
+ return "byte"
+ elif bits == 16:
+ return "short"
+ elif bits == 32:
+ return "int"
+ elif bits == 64:
+ return "long"
+ else:
+ assert False
def packed64singleblock(bpv, f):
values = 64 / bpv
- f.write("\n static final class Packed64SingleBlockBulkOperation%d extends BulkOperation {\n\n" %bpv)
- f.write(" public int blocks() {\n")
+ f.write(" @Override\n")
+ f.write(" public int blockCount() {\n")
f.write(" return 1;\n")
f.write(" }\n\n")
- f.write(" public int values() {\n")
+ f.write(" @Override\n")
+ f.write(" public int valueCount() {\n")
f.write(" return %d;\n" %values)
f.write(" }\n\n")
+ p64sb_decode(bpv, f, 32)
+ p64sb_decode(bpv, f, 64)
+ p64sb_encode(bpv, f, 32)
+ p64sb_encode(bpv, f, 64)
- f.write(" public void get(long[] blocks, int bi, long[] values, int vi, int iterations) {\n")
- f.write(" assert bi + iterations * blocks() <= blocks.length;\n")
- f.write(" assert vi + iterations * values() <= values.length;\n")
+def p64sb_decode(bpv, f, bits):
+ values = 64 / bpv
+ typ = get_type(bits)
+ cast_start, cast_end = casts(typ)
+ f.write(" @Override\n")
+ f.write(" public void decode(long[] blocks, int blocksOffset, %s[] values, int valuesOffset, int iterations) {\n" %typ)
+ if bits < bpv:
+ f.write(" throw new UnsupportedOperationException();\n")
+ f.write(" }\n\n")
+ return
+ f.write(" assert blocksOffset + iterations * blockCount() <= blocks.length;\n")
+ f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
- f.write(" final long block = blocks[bi++];\n")
+ f.write(" final long block = blocks[blocksOffset++];\n")
mask = (1 << bpv) - 1
for i in xrange(values):
block_offset = i / values
offset_in_block = i % values
if i == 0:
- f.write(" values[vi++] = block & %dL;\n" %mask)
+ f.write(" values[valuesOffset++] = %sblock & %dL%s;\n" %(cast_start, mask, cast_end))
elif i == values - 1:
- f.write(" values[vi++] = block >>> %d;\n" %(i * bpv))
+ f.write(" values[valuesOffset++] = %sblock >>> %d%s;\n" %(cast_start, i * bpv, cast_end))
else:
- f.write(" values[vi++] = (block >>> %d) & %dL;\n" %(i * bpv, mask))
+ f.write(" values[valuesOffset++] = %s(block >>> %d) & %dL%s;\n" %(cast_start, i * bpv, mask, cast_end))
f.write(" }\n")
f.write(" }\n\n")
- f.write(" public void set(long[] blocks, int bi, long[] values, int vi, int iterations) {\n")
- f.write(" assert bi + iterations * blocks() <= blocks.length;\n")
- f.write(" assert vi + iterations * values() <= values.length;\n")
+ f.write(" @Override\n")
+ f.write(" public void decode(byte[] blocks, int blocksOffset, %s[] values, int valuesOffset, int iterations) {\n" %typ)
+ if bits < bpv:
+ f.write(" throw new UnsupportedOperationException();\n")
+ f.write(" }\n\n")
+ f.write(" assert blocksOffset + 8 * iterations * blockCount() <= blocks.length;\n")
+ f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
+ f.write(" for (int i = 0; i < iterations; ++i) {\n")
+ if bpv >= 32 and bits > 32:
+ for i in xrange(7, -1, -1):
+ f.write(" final long byte%d = blocks[blocksOffset++] & 0xFF;\n" %i)
+ else:
+ for i in xrange(7, -1, -1):
+ f.write(" final int byte%d = blocks[blocksOffset++] & 0xFF;\n" %i)
+ for i in xrange(values):
+ byte_start = (i * bpv) / 8
+ bit_start = (i * bpv) % 8
+ byte_end = ((i + 1) * bpv - 1) / 8
+ bit_end = ((i + 1) * bpv - 1) % 8
+ f.write(" values[valuesOffset++] =")
+ if byte_start == byte_end:
+ # only one byte
+ if bit_start == 0:
+ if bit_end == 7:
+ f.write(" byte%d" %byte_start)
+ else:
+ f.write(" byte%d & %d" %(byte_start, mask))
+ else:
+ if bit_end == 7:
+ f.write(" byte%d >>> %d" %(byte_start, bit_start))
+ else:
+ f.write(" (byte%d >>> %d) & %d" %(byte_start, bit_start, mask))
+ else:
+ if bit_start == 0:
+ f.write(" byte%d" %byte_start)
+ else:
+ f.write(" (byte%d >>> %d)" %(byte_start, bit_start))
+ for b in xrange(byte_start + 1, byte_end):
+ f.write(" | (byte%d << %d)" %(b, 8 * (b - byte_start) - bit_start))
+ if bit_end == 7:
+ f.write(" | (byte%d << %d)" %(byte_end, 8 * (byte_end - byte_start) - bit_start))
+ else:
+ f.write(" | ((byte%d & %d) << %d)" %(byte_end, 2 ** (bit_end + 1) - 1, 8 * (byte_end - byte_start) - bit_start))
+ f.write(";\n")
+ f.write(" }\n")
+ f.write(" }\n\n")
+
+def p64sb_encode(bpv, f, bits):
+ values = 64 / bpv
+ typ = get_type(bits)
+ mask_start, mask_end = masks(bits)
+ f.write(" @Override\n")
+ f.write(" public void encode(%s[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {\n" %typ)
+ if bits < bpv:
+ f.write(" throw new UnsupportedOperationException();\n")
+ f.write(" }\n\n")
+ return
+ f.write(" assert blocksOffset + iterations * blockCount() <= blocks.length;\n")
+ f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
for i in xrange(values):
block_offset = i / values
offset_in_block = i % values
if i == 0:
- f.write(" blocks[bi++] = values[vi++]")
+ f.write(" blocks[blocksOffset++] = %svalues[valuesOffset++]%s" %(mask_start, mask_end))
else:
- f.write(" | (values[vi++] << %d)" %(i * bpv))
+ f.write(" | (%svalues[valuesOffset++]%s << %d)" %(mask_start, mask_end, i * bpv))
if i == values - 1:
f.write(";\n")
f.write(" }\n")
- f.write(" }\n")
-
- f.write(" }\n")
+ f.write(" }\n\n")
def packed64(bpv, f):
blocks = bpv
@@ -169,96 +260,223 @@ def packed64(bpv, f):
values /= 2
assert values * bpv == 64 * blocks, "%d values, %d blocks, %d bits per value" %(values, blocks, bpv)
mask = (1 << bpv) - 1
- f.write(" static final class Packed64BulkOperation%d extends BulkOperation {\n\n" %bpv)
- f.write(" public int blocks() {\n")
+ f.write(" @Override\n")
+ f.write(" public int blockCount() {\n")
f.write(" return %d;\n" %blocks)
f.write(" }\n\n")
- f.write(" public int values() {\n")
+ f.write(" @Override\n")
+ f.write(" public int valueCount() {\n")
f.write(" return %d;\n" %values)
f.write(" }\n\n")
if bpv == 64:
- f.write(""" public void get(long[] blocks, int bi, long[] values, int vi, int iterations) {
- System.arraycopy(blocks, bi, values, vi, iterations);
+ f.write(""" @Override
+ public void decode(long[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
+ System.arraycopy(blocks, blocksOffset, values, valuesOffset, valueCount() * iterations);
}
- public void set(long[] blocks, int bi, long[] values, int vi, int iterations) {
- System.arraycopy(values, bi, blocks, vi, iterations);
+ @Override
+ public void decode(long[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
+ LongBuffer.wrap(values, valuesOffset, iterations * valueCount()).put(ByteBuffer.wrap(blocks, blocksOffset, 8 * iterations * blockCount()).asLongBuffer());
+ }
+
+ @Override
+ public void encode(long[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {
+ System.arraycopy(values, valuesOffset, blocks, blocksOffset, valueCount() * iterations);
}
- }
""")
- return
+ else:
+ p64_decode(bpv, f, 32, values)
+ p64_decode(bpv, f, 64, values)
+ p64_encode(bpv, f, 32, values)
+ p64_encode(bpv, f, 64, values)
+
+def p64_decode(bpv, f, bits, values):
+ typ = get_type(bits)
+ cast_start, cast_end = casts(typ)
+
+ f.write(" @Override\n")
+ f.write(" public void decode(long[] blocks, int blocksOffset, %s[] values, int valuesOffset, int iterations) {\n" %typ)
+ if bits < bpv:
+ f.write(" throw new UnsupportedOperationException();\n")
+ else:
+ f.write(" assert blocksOffset + iterations * blockCount() <= blocks.length;\n")
+ f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
+ f.write(" for (int i = 0; i < iterations; ++i) {\n")
+ mask = (1 << bpv) - 1
+ for i in xrange(0, values):
+ block_offset = i * bpv / 64
+ bit_offset = (i * bpv) % 64
+ if bit_offset == 0:
+ # start of block
+ f.write(" final long block%d = blocks[blocksOffset++];\n" %block_offset);
+ f.write(" values[valuesOffset++] = %sblock%d >>> %d%s;\n" %(cast_start, block_offset, 64 - bpv, cast_end))
+ elif bit_offset + bpv == 64:
+ # end of block
+ f.write(" values[valuesOffset++] = %sblock%d & %dL%s;\n" %(cast_start, block_offset, mask, cast_end))
+ elif bit_offset + bpv < 64:
+ # middle of block
+ f.write(" values[valuesOffset++] = %s(block%d >>> %d) & %dL%s;\n" %(cast_start, block_offset, 64 - bit_offset - bpv, mask, cast_end))
+ else:
+ # value spans across 2 blocks
+ mask1 = (1 << (64 - bit_offset)) -1
+ shift1 = bit_offset + bpv - 64
+ shift2 = 64 - shift1
+ f.write(" final long block%d = blocks[blocksOffset++];\n" %(block_offset + 1));
+ f.write(" values[valuesOffset++] = %s((block%d & %dL) << %d) | (block%d >>> %d)%s;\n" %(cast_start, block_offset, mask1, shift1, block_offset + 1, shift2, cast_end))
+ f.write(" }\n")
+ f.write(" }\n\n")
- f.write(" public void get(long[] blocks, int bi, long[] values, int vi, int iterations) {\n")
- f.write(" assert bi + iterations * blocks() <= blocks.length;\n")
- f.write(" assert vi + iterations * values() <= values.length;\n")
- f.write(" for (int i = 0; i < iterations; ++i) {\n")
- for i in xrange(0, values):
- block_offset = i * bpv / 64
- bit_offset = (i * bpv) % 64
- if bit_offset == 0:
- # start of block
- f.write(" final long block%d = blocks[bi++];\n" %block_offset);
- f.write(" values[vi++] = block%d >>> %d;\n" %(block_offset, 64 - bpv))
- elif bit_offset + bpv == 64:
- # end of block
- f.write(" values[vi++] = block%d & %dL;\n" %(block_offset, mask))
- elif bit_offset + bpv < 64:
- # middle of block
- f.write(" values[vi++] = (block%d >>> %d) & %dL;\n" %(block_offset, 64 - bit_offset - bpv, mask))
- else:
- # value spans across 2 blocks
- mask1 = (1 << (64 - bit_offset)) -1
- shift1 = bit_offset + bpv - 64
- shift2 = 64 - shift1
- f.write(" final long block%d = blocks[bi++];\n" %(block_offset + 1));
- f.write(" values[vi++] = ((block%d & %dL) << %d) | (block%d >>> %d);\n" %(block_offset, mask1, shift1, block_offset + 1, shift2))
- f.write(" }\n")
+ f.write(" @Override\n")
+ f.write(" public void decode(byte[] blocks, int blocksOffset, %s[] values, int valuesOffset, int iterations) {\n" %typ)
+ if bits < bpv:
+ f.write(" throw new UnsupportedOperationException();\n")
+ else:
+ f.write(" assert blocksOffset + 8 * iterations * blockCount() <= blocks.length;\n")
+ f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
+ f.write(" for (int i = 0; i < iterations; ++i) {\n")
+ blocks = values * bpv / 8
+ for i in xrange(0, values):
+ byte_start = i * bpv / 8
+ bit_start = (i * bpv) % 8
+ byte_end = ((i + 1) * bpv - 1) / 8
+ bit_end = ((i + 1) * bpv - 1) % 8
+ shift = lambda b: 8 * (byte_end - b - 1) + 1 + bit_end
+ if bit_start == 0:
+ f.write(" final %s byte%d = blocks[blocksOffset++] & 0xFF;\n" %(typ, byte_start))
+ for b in xrange(byte_start + 1, byte_end + 1):
+ f.write(" final %s byte%d = blocks[blocksOffset++] & 0xFF;\n" %(typ, b))
+ f.write(" values[valuesOffset++] =")
+ if byte_start == byte_end:
+ if bit_start == 0:
+ if bit_end == 7:
+ f.write(" byte%d" %byte_start)
+ else:
+ f.write(" byte%d >>> %d" %(byte_start, 7 - bit_end))
+ else:
+ if bit_end == 7:
+ f.write(" byte%d & %d" %(byte_start, 2 ** (8 - bit_start) - 1))
+ else:
+ f.write(" (byte%d >>> %d) & %d" %(byte_start, 7 - bit_end, 2 ** (bit_end - bit_start + 1) - 1))
+ else:
+ if bit_start == 0:
+ f.write(" (byte%d << %d)" %(byte_start, shift(byte_start)))
+ else:
+ f.write(" ((byte%d & %d) << %d)" %(byte_start, 2 ** (8 - bit_start) - 1, shift(byte_start)))
+ for b in xrange(byte_start + 1, byte_end):
+ f.write(" | (byte%d << %d)" %(b, shift(b)))
+ if bit_end == 7:
+ f.write(" | byte%d" %byte_end)
+ else:
+ f.write(" | (byte%d >>> %d)" %(byte_end, 7 - bit_end))
+ f.write(";\n")
+ f.write(" }\n")
f.write(" }\n\n")
- f.write(" public void set(long[] blocks, int bi, long[] values, int vi, int iterations) {\n")
- f.write(" assert bi + iterations * blocks() <= blocks.length;\n")
- f.write(" assert vi + iterations * values() <= values.length;\n")
+def p64_encode(bpv, f, bits, values):
+ typ = get_type(bits)
+ mask_start, mask_end = masks(bits)
+ f.write(" @Override\n")
+ f.write(" public void encode(%s[] values, int valuesOffset, long[] blocks, int blocksOffset, int iterations) {\n" %typ)
+ f.write(" assert blocksOffset + iterations * blockCount() <= blocks.length;\n")
+ f.write(" assert valuesOffset + iterations * valueCount() <= values.length;\n")
f.write(" for (int i = 0; i < iterations; ++i) {\n")
for i in xrange(0, values):
block_offset = i * bpv / 64
bit_offset = (i * bpv) % 64
if bit_offset == 0:
# start of block
- f.write(" blocks[bi++] = (values[vi++] << %d)" %(64 - bpv))
+ f.write(" blocks[blocksOffset++] = (%svalues[valuesOffset++]%s << %d)" %(mask_start, mask_end, 64 - bpv))
elif bit_offset + bpv == 64:
# end of block
- f.write(" | values[vi++];\n")
+ f.write(" | %svalues[valuesOffset++]%s;\n" %(mask_start, mask_end))
elif bit_offset + bpv < 64:
# inside a block
- f.write(" | (values[vi++] << %d)" %(64 - bit_offset - bpv))
+ f.write(" | (%svalues[valuesOffset++]%s << %d)" %(mask_start, mask_end, 64 - bit_offset - bpv))
else:
# value spans across 2 blocks
right_bits = bit_offset + bpv - 64
- f.write(" | (values[vi] >>> %d);\n" %right_bits)
- f.write(" blocks[bi++] = (values[vi++] << %d)" %(64 - right_bits))
+ f.write(" | (%svalues[valuesOffset]%s >>> %d);\n" %(mask_start, mask_end, right_bits))
+ f.write(" blocks[blocksOffset++] = (%svalues[valuesOffset++]%s << %d)" %(mask_start, mask_end, 64 - right_bits))
f.write(" }\n")
- f.write(" }\n")
-
- f.write(" }\n\n")
-
+ f.write(" }\n\n")
if __name__ == '__main__':
p64_bpv = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 12, 16, 21, 32]
+
f = open(OUTPUT_FILE, 'w')
f.write(HEADER)
- f.write(" static {\n")
- f.write(" BULK_OPERATIONS.put(PackedInts.Format.PACKED, new BulkOperation[65]);")
+ f.write('import java.nio.ByteBuffer;\n')
+ f.write('\n')
+ f.write('''/**
+ * Efficient sequential read/write of packed integers.
+ */\n''')
+
+ f.write('abstract class BulkOperation implements PackedInts.Decoder, PackedInts.Encoder {\n')
+ f.write(' private static final BulkOperation[] packedBulkOps = new BulkOperation[] {\n')
+
for bpv in xrange(1, 65):
- f.write(" BULK_OPERATIONS.get(PackedInts.Format.PACKED)[%d] = new Packed64BulkOperation%d();\n" %(bpv, bpv))
- f.write(" BULK_OPERATIONS.put(PackedInts.Format.PACKED_SINGLE_BLOCK, new BulkOperation[65]);\n")
- for bpv in PACKED_64_SINGLE_BLOCK_BPV:
- f.write(" BULK_OPERATIONS.get(PackedInts.Format.PACKED_SINGLE_BLOCK)[%d] = new Packed64SingleBlockBulkOperation%d();\n" %(bpv, bpv))
+ f2 = open('BulkOperationPacked%d.java' % bpv, 'w')
+ f2.write(HEADER)
+ if bpv == 64:
+ f2.write('import java.nio.LongBuffer;\n')
+ f2.write('import java.nio.ByteBuffer;\n')
+ f2.write('\n')
+ f2.write('''/**
+ * Efficient sequential read/write of packed integers.
+ */\n''')
+ f2.write('final class BulkOperationPacked%d extends BulkOperation {\n' % bpv)
+ packed64(bpv, f2)
+ f2.write('}\n')
+ f2.close()
+ f.write(' new BulkOperationPacked%d(),\n' % bpv)
+
+ f.write(' };\n')
+ f.write('\n')
+
+ f.write(' // NOTE: this is sparse (some entries are null):\n')
+ f.write(' private static final BulkOperation[] packedSingleBlockBulkOps = new BulkOperation[] {\n')
+ for bpv in xrange(1, max(PACKED_64_SINGLE_BLOCK_BPV)+1):
+ if bpv in PACKED_64_SINGLE_BLOCK_BPV:
+ f2 = open('BulkOperationPackedSingleBlock%d.java' % bpv, 'w')
+ f2.write(HEADER)
+ f2.write('''/**
+ * Efficient sequential read/write of packed integers.
+ */\n''')
+ f2.write('final class BulkOperationPackedSingleBlock%d extends BulkOperation {\n' % bpv)
+ packed64singleblock(bpv,f2)
+ f2.write('}\n')
+ f2.close()
+ f.write(' new BulkOperationPackedSingleBlock%d(),\n' % bpv)
+ else:
+ f.write(' null,\n')
+ f.write(' };\n')
+ f.write('\n')
+
+ f.write("\n")
+ f.write(" public static BulkOperation of(PackedInts.Format format, int bitsPerValue) {\n")
+ f.write(" switch (format) {\n")
+
+ f.write(" case PACKED:\n")
+ f.write(" assert packedBulkOps[bitsPerValue - 1] != null;\n")
+ f.write(" return packedBulkOps[bitsPerValue - 1];\n")
+ f.write(" case PACKED_SINGLE_BLOCK:\n")
+ f.write(" assert packedSingleBlockBulkOps[bitsPerValue - 1] != null;\n")
+ f.write(" return packedSingleBlockBulkOps[bitsPerValue - 1];\n")
+ f.write(" default:\n")
+ f.write(" throw new AssertionError();\n")
+ f.write(" }\n")
f.write(" }\n")
- for bpv in xrange(1, 65):
- packed64(bpv, f)
- for bpv in PACKED_64_SINGLE_BLOCK_BPV:
- packed64singleblock(bpv,f)
f.write(FOOTER)
f.close()
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/gen_Packed64SingleBlock.py Sat Aug 25 10:06:07 2012
@@ -105,12 +105,12 @@ abstract class Packed64SingleBlock exten
// bulk get
assert index %% valuesPerBlock == 0;
- final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
- assert op.blocks() == 1;
- assert op.values() == valuesPerBlock;
+ final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
+ assert decoder.blockCount() == 1;
+ assert decoder.valueCount() == valuesPerBlock;
final int blockIndex = index / valuesPerBlock;
final int nblocks = (index + len) / valuesPerBlock - blockIndex;
- op.get(blocks, blockIndex, arr, off, nblocks);
+ decoder.decode(blocks, blockIndex, arr, off, nblocks);
final int diff = nblocks * valuesPerBlock;
index += diff; len -= diff;
@@ -150,11 +150,11 @@ abstract class Packed64SingleBlock exten
// bulk set
assert index %% valuesPerBlock == 0;
final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
- assert op.blocks() == 1;
- assert op.values() == valuesPerBlock;
+ assert op.blockCount() == 1;
+ assert op.valueCount() == valuesPerBlock;
final int blockIndex = index / valuesPerBlock;
final int nblocks = (index + len) / valuesPerBlock - blockIndex;
- op.set(blocks, blockIndex, arr, off, nblocks);
+ op.encode(arr, off, blocks, blockIndex, nblocks);
final int diff = nblocks * valuesPerBlock;
index += diff; len -= diff;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Aug 25 10:06:07 2012
@@ -19,3 +19,4 @@ org.apache.lucene.codecs.simpletext.Simp
org.apache.lucene.codecs.memory.MemoryPostingsFormat
org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat
org.apache.lucene.codecs.memory.DirectPostingsFormat
+org.apache.lucene.codecs.block.BlockPostingsFormat
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestCharFilter.java Sat Aug 25 10:06:07 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.analysis;
* limitations under the License.
*/
+import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
@@ -51,6 +52,11 @@ public class TestCharFilter extends Luce
}
@Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ return input.read(cbuf, off, len);
+ }
+
+ @Override
protected int correct(int currentOff) {
return currentOff + 1;
}
@@ -61,6 +67,11 @@ public class TestCharFilter extends Luce
protected CharFilter2(Reader in) {
super(in);
}
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ return input.read(cbuf, off, len);
+ }
@Override
protected int correct(int currentOff) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/document/TestField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/document/TestField.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/document/TestField.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/document/TestField.java Sat Aug 25 10:06:07 2012
@@ -18,9 +18,11 @@ package org.apache.lucene.document;
*/
import java.io.StringReader;
+import java.nio.CharBuffer;
import org.apache.lucene.analysis.CannedTokenStream;
import org.apache.lucene.analysis.Token;
+import org.apache.lucene.document.Field.ReusableStringReader;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -610,4 +612,40 @@ public class TestField extends LuceneTes
// expected
}
}
+
+ public void testReusableStringReader() throws Exception {
+ ReusableStringReader reader = new ReusableStringReader();
+ assertEquals(-1, reader.read());
+ assertEquals(-1, reader.read(new char[1]));
+ assertEquals(-1, reader.read(new char[2], 1, 1));
+ assertEquals(-1, reader.read(CharBuffer.wrap(new char[2])));
+
+ reader.setValue("foobar");
+ char[] buf = new char[4];
+ assertEquals(4, reader.read(buf));
+ assertEquals("foob", new String(buf));
+ assertEquals(2, reader.read(buf));
+ assertEquals("ar", new String(buf, 0, 2));
+ assertEquals(-1, reader.read(buf));
+ reader.close();
+
+ reader.setValue("foobar");
+ assertEquals(0, reader.read(buf, 1, 0));
+ assertEquals(3, reader.read(buf, 1, 3));
+ assertEquals("foo", new String(buf, 1, 3));
+ assertEquals(2, reader.read(CharBuffer.wrap(buf, 2, 2)));
+ assertEquals("ba", new String(buf, 2, 2));
+ assertEquals('r', (char) reader.read());
+ assertEquals(-1, reader.read(buf));
+ reader.close();
+
+ reader.setValue("foobar");
+ StringBuilder sb = new StringBuilder();
+ int ch;
+ while ((ch = reader.read()) != -1) {
+ sb.append((char) ch);
+ }
+ reader.close();
+ assertEquals("foobar", sb.toString());
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestCompoundFile.java Sat Aug 25 10:06:07 2012
@@ -350,7 +350,7 @@ public class TestCompoundFile extends Lu
IndexInput one = cr.openInput("f11", newIOContext(random()));
- IndexInput two = (IndexInput) one.clone();
+ IndexInput two = one.clone();
assertSameStreams("basic clone one", expected, one);
expected.seek(0);
@@ -476,8 +476,8 @@ public class TestCompoundFile extends Lu
IndexInput e1 = cr.openInput("f11", newIOContext(random()));
IndexInput e2 = cr.openInput("f3", newIOContext(random()));
- IndexInput a1 = (IndexInput) e1.clone();
- IndexInput a2 = (IndexInput) e2.clone();
+ IndexInput a1 = e1.clone();
+ IndexInput a2 = e2.clone();
// Seek the first pair
e1.seek(100);
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java Sat Aug 25 10:06:07 2012
@@ -807,7 +807,8 @@ public void testFilesOpenClose() throws
DirectoryReader r = DirectoryReader.open(dir);
AtomicReader r1 = getOnlySegmentReader(r);
- assertEquals(36, r1.getUniqueTermCount());
+ assertEquals(26, r1.terms("field").size());
+ assertEquals(10, r1.terms("number").size());
writer.addDocument(doc);
writer.commit();
DirectoryReader r2 = DirectoryReader.openIfChanged(r);
@@ -815,7 +816,8 @@ public void testFilesOpenClose() throws
r.close();
for(AtomicReaderContext s : r2.leaves()) {
- assertEquals(36, s.reader().getUniqueTermCount());
+ assertEquals(26, s.reader().terms("field").size());
+ assertEquals(10, s.reader().terms("number").size());
}
r2.close();
writer.close();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sat Aug 25 10:06:07 2012
@@ -191,10 +191,6 @@ public class TestDuelingCodecs extends L
if (leftFields.size() != -1 && rightFields.size() != -1) {
assertEquals(info, leftFields.size(), rightFields.size());
}
-
- if (leftFields.getUniqueTermCount() != -1 && rightFields.getUniqueTermCount() != -1) {
- assertEquals(info, leftFields.getUniqueTermCount(), rightFields.getUniqueTermCount());
- }
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestFieldsReader.java Sat Aug 25 10:06:07 2012
@@ -186,7 +186,7 @@ public class TestFieldsReader extends Lu
}
@Override
public FaultyIndexInput clone() {
- return new FaultyIndexInput((IndexInput) delegate.clone());
+ return new FaultyIndexInput(delegate.clone());
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java Sat Aug 25 10:06:07 2012
@@ -263,4 +263,31 @@ public class TestIndexWriterMergePolicy
assertTrue(numSegments < mergeFactor);
}
}
+
+ private static final double EPSILON = 1E-14;
+
+ public void testSetters() {
+ assertSetters(new LogByteSizeMergePolicy());
+ assertSetters(new LogDocMergePolicy());
+ }
+
+ private void assertSetters(LogMergePolicy lmp) {
+ lmp.setMaxCFSSegmentSizeMB(2.0);
+ assertEquals(2.0, lmp.getMaxCFSSegmentSizeMB(), EPSILON);
+
+ lmp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
+ assertEquals(Long.MAX_VALUE/1024/1024., lmp.getMaxCFSSegmentSizeMB(), EPSILON*Long.MAX_VALUE);
+
+ lmp.setMaxCFSSegmentSizeMB(Long.MAX_VALUE/1024/1024.);
+ assertEquals(Long.MAX_VALUE/1024/1024., lmp.getMaxCFSSegmentSizeMB(), EPSILON*Long.MAX_VALUE);
+
+ try {
+ lmp.setMaxCFSSegmentSizeMB(-2.0);
+ fail("Didn't throw IllegalArgumentException");
+ } catch (IllegalArgumentException iae) {
+ // pass
+ }
+
+ // TODO: Add more checks for other non-double setters!
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Sat Aug 25 10:06:07 2012
@@ -249,7 +249,7 @@ public class TestIndexWriterOnDiskFull e
// Make a new dir that will enforce disk usage:
MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new RAMDirectory(startDir, newIOContext(random())));
- writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy()));
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND).setMergePolicy(newLogMergePolicy(false)));
IOException err = null;
MergeScheduler ms = writer.getConfig().getMergeScheduler();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java Sat Aug 25 10:06:07 2012
@@ -226,7 +226,7 @@ public class TestLazyProxSkipping extend
@Override
public SeeksCountingStream clone() {
- return new SeeksCountingStream((IndexInput) this.input.clone());
+ return new SeeksCountingStream(this.input.clone());
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java Sat Aug 25 10:06:07 2012
@@ -190,7 +190,7 @@ public class TestMultiLevelSkipList exte
@Override
public CountingStream clone() {
- return new CountingStream((IndexInput) this.input.clone());
+ return new CountingStream(this.input.clone());
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java Sat Aug 25 10:06:07 2012
@@ -933,6 +933,7 @@ public class TestPostingsFormat extends
// NOTE: you can also test "weaker" index options than
// you indexed with:
testTerms(fieldsProducer, EnumSet.allOf(Option.class), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
+ //testTerms(fieldsProducer, EnumSet.complementOf(EnumSet.of(Option.THREADS)), IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
fieldsProducer.close();
dir.close();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Sat Aug 25 10:06:07 2012
@@ -444,9 +444,9 @@ public class TestPostingsOffsets extends
makeToken("foo", 1, 0, 3),
makeToken("foo", 0, 0, 3),
makeToken("foo", 0, 0, 3)
- });
+ });
}
-
+
public void testLegalbutVeryLargeOffsets() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestTieredMergePolicy.java Sat Aug 25 10:06:07 2012
@@ -153,4 +153,60 @@ public class TestTieredMergePolicy exten
dir.close();
}
+
+ private static final double EPSILON = 1E-14;
+
+ public void testSetters() {
+ final TieredMergePolicy tmp = new TieredMergePolicy();
+
+ tmp.setMaxMergedSegmentMB(0.5);
+ assertEquals(0.5, tmp.getMaxMergedSegmentMB(), EPSILON);
+
+ tmp.setMaxMergedSegmentMB(Double.POSITIVE_INFINITY);
+ assertEquals(Long.MAX_VALUE/1024/1024., tmp.getMaxMergedSegmentMB(), EPSILON*Long.MAX_VALUE);
+
+ tmp.setMaxMergedSegmentMB(Long.MAX_VALUE/1024/1024.);
+ assertEquals(Long.MAX_VALUE/1024/1024., tmp.getMaxMergedSegmentMB(), EPSILON*Long.MAX_VALUE);
+
+ try {
+ tmp.setMaxMergedSegmentMB(-2.0);
+ fail("Didn't throw IllegalArgumentException");
+ } catch (IllegalArgumentException iae) {
+ // pass
+ }
+
+ tmp.setFloorSegmentMB(2.0);
+ assertEquals(2.0, tmp.getFloorSegmentMB(), EPSILON);
+
+ tmp.setFloorSegmentMB(Double.POSITIVE_INFINITY);
+ assertEquals(Long.MAX_VALUE/1024/1024., tmp.getFloorSegmentMB(), EPSILON*Long.MAX_VALUE);
+
+ tmp.setFloorSegmentMB(Long.MAX_VALUE/1024/1024.);
+ assertEquals(Long.MAX_VALUE/1024/1024., tmp.getFloorSegmentMB(), EPSILON*Long.MAX_VALUE);
+
+ try {
+ tmp.setFloorSegmentMB(-2.0);
+ fail("Didn't throw IllegalArgumentException");
+ } catch (IllegalArgumentException iae) {
+ // pass
+ }
+
+ tmp.setMaxCFSSegmentSizeMB(2.0);
+ assertEquals(2.0, tmp.getMaxCFSSegmentSizeMB(), EPSILON);
+
+ tmp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
+ assertEquals(Long.MAX_VALUE/1024/1024., tmp.getMaxCFSSegmentSizeMB(), EPSILON*Long.MAX_VALUE);
+
+ tmp.setMaxCFSSegmentSizeMB(Long.MAX_VALUE/1024/1024.);
+ assertEquals(Long.MAX_VALUE/1024/1024., tmp.getMaxCFSSegmentSizeMB(), EPSILON*Long.MAX_VALUE);
+
+ try {
+ tmp.setMaxCFSSegmentSizeMB(-2.0);
+ fail("Didn't throw IllegalArgumentException");
+ } catch (IllegalArgumentException iae) {
+ // pass
+ }
+
+ // TODO: Add more checks for other non-double setters!
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestCopyBytes.java Sat Aug 25 10:06:07 2012
@@ -127,7 +127,7 @@ public class TestCopyBytes extends Lucen
// now make N copies of the remaining bytes
CopyThread copies[] = new CopyThread[10];
for (int i = 0; i < copies.length; i++) {
- copies[i] = new CopyThread((IndexInput) input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT));
+ copies[i] = new CopyThread(input.clone(), d.createOutput("copy" + i, IOContext.DEFAULT));
}
for (int i = 0; i < copies.length; i++) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/store/TestMultiMMap.java Sat Aug 25 10:06:07 2012
@@ -53,8 +53,8 @@ public class TestMultiMMap extends Lucen
io.writeVInt(5);
io.close();
IndexInput one = mmapDir.openInput("bytes", IOContext.DEFAULT);
- IndexInput two = (IndexInput) one.clone();
- IndexInput three = (IndexInput) two.clone(); // clone of clone
+ IndexInput two = one.clone();
+ IndexInput three = two.clone(); // clone of clone
one.close();
try {
one.readVInt();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/TestMaxFailuresRule.java Sat Aug 25 10:06:07 2012
@@ -19,9 +19,9 @@ package org.apache.lucene.util;
import org.apache.lucene.util.junitcompat.WithNestedTests;
import org.junit.Assert;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
+import org.junit.runner.Description;
import org.junit.runner.JUnitCore;
import org.junit.runner.Result;
import org.junit.runner.notification.Failure;
@@ -45,7 +45,7 @@ public class TestMaxFailuresRule extends
}
public static class Nested extends WithNestedTests.AbstractNestedTest {
- @Repeat(iterations = 100)
+ @Repeat(iterations = 500)
public void testFailSometimes() {
assertFalse(random().nextInt(5) == 0);
}
@@ -61,22 +61,40 @@ public class TestMaxFailuresRule extends
LuceneTestCase.ignoreAfterMaxFailures.failuresSoFar = 0;
JUnitCore core = new JUnitCore();
- final int [] assumptions = new int [1];
+ final StringBuilder results = new StringBuilder();
core.addListener(new RunListener() {
+ char lastTest;
+
+ @Override
+ public void testStarted(Description description) throws Exception {
+ lastTest = 'S'; // success.
+ }
+
@Override
public void testAssumptionFailure(Failure failure) {
- assumptions[0]++;
+ lastTest = 'A'; // assumption failure.
+ }
+
+ @Override
+ public void testFailure(Failure failure) throws Exception {
+ lastTest = 'F'; // failure
+ }
+
+ @Override
+ public void testFinished(Description description) throws Exception {
+ results.append(lastTest);
}
});
Result result = core.run(Nested.class);
- Assert.assertEquals(100, result.getRunCount());
+ Assert.assertEquals(500, result.getRunCount());
Assert.assertEquals(0, result.getIgnoreCount());
Assert.assertEquals(2, result.getFailureCount());
- // JUnit doesn't pass back the number of successful tests, just make sure
- // we did have enough assumption-failures.
- Assert.assertTrue(assumptions[0] > 50);
+ // Make sure we had exactly two failures followed by assumption-failures
+ // resulting from ignored tests.
+ Assert.assertTrue(results.toString(),
+ results.toString().matches("(S*F){2}A+"));
} finally {
LuceneTestCase.ignoreAfterMaxFailures.maxFailures = maxFailures;
LuceneTestCase.ignoreAfterMaxFailures.failuresSoFar = failuresSoFar;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Sat Aug 25 10:06:07 2012
@@ -18,17 +18,24 @@ package org.apache.lucene.util.packed;
*/
import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.LongBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Locale;
import java.util.Random;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.store.*;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.LuceneTestCase.Slow;
+import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.packed.PackedInts.Reader;
@Slow
@@ -622,4 +629,106 @@ public class TestPackedInts extends Luce
}
}
+ public void testEncodeDecode() {
+ for (PackedInts.Format format : PackedInts.Format.values()) {
+ for (int bpv = 1; bpv <= 64; ++bpv) {
+ if (!format.isSupported(bpv)) {
+ continue;
+ }
+ String msg = format + " " + bpv;
+
+ final PackedInts.Encoder encoder = PackedInts.getEncoder(format, PackedInts.VERSION_CURRENT, bpv);
+ final PackedInts.Decoder decoder = PackedInts.getDecoder(format, PackedInts.VERSION_CURRENT, bpv);
+ final int blockCount = encoder.blockCount();
+ final int valueCount = encoder.valueCount();
+ assertEquals(blockCount, decoder.blockCount());
+ assertEquals(valueCount, decoder.valueCount());
+
+ final int iterations = random().nextInt(100);
+ final int blocksOffset = random().nextInt(100);
+ final int valuesOffset = random().nextInt(100);
+ final int blocksOffset2 = random().nextInt(100);
+ final int blocksLen = iterations * blockCount;
+
+ // 1. generate random inputs
+ final long[] blocks = new long[blocksOffset + blocksLen];
+ for (int i = 0; i < blocks.length; ++i) {
+ blocks[i] = random().nextLong();
+ if (format == PackedInts.Format.PACKED_SINGLE_BLOCK && 64 % bpv != 0) {
+ // clear highest bits for packed
+ final int toClear = 64 % bpv;
+ blocks[i] = (blocks[i] << toClear) >>> toClear;
+ }
+ }
+
+ // 2. decode
+ final long[] values = new long[valuesOffset + iterations * valueCount];
+ decoder.decode(blocks, blocksOffset, values, valuesOffset, iterations);
+ for (long value : values) {
+ assertTrue(value <= PackedInts.maxValue(bpv));
+ }
+ // test decoding to int[]
+ final int[] intValues;
+ if (bpv <= 32) {
+ intValues = new int[values.length];
+ decoder.decode(blocks, blocksOffset, intValues, valuesOffset, iterations);
+ assertTrue(equals(intValues, values));
+ } else {
+ intValues = null;
+ }
+
+ // 3. re-encode
+ final long[] blocks2 = new long[blocksOffset2 + blocksLen];
+ encoder.encode(values, valuesOffset, blocks2, blocksOffset2, iterations);
+ assertArrayEquals(msg, Arrays.copyOfRange(blocks, blocksOffset, blocks.length),
+ Arrays.copyOfRange(blocks2, blocksOffset2, blocks2.length));
+ // test encoding from int[]
+ if (bpv <= 32) {
+ final long[] blocks3 = new long[blocks2.length];
+ encoder.encode(intValues, valuesOffset, blocks3, blocksOffset2, iterations);
+ assertArrayEquals(msg, blocks2, blocks3);
+ }
+
+ // 4. byte[] decoding
+ final byte[] byteBlocks = new byte[8 * blocks.length];
+ ByteBuffer.wrap(byteBlocks).asLongBuffer().put(blocks);
+ final long[] values2 = new long[valuesOffset + iterations * valueCount];
+ decoder.decode(byteBlocks, blocksOffset * 8, values2, valuesOffset, iterations);
+ for (long value : values2) {
+ assertTrue(msg, value <= PackedInts.maxValue(bpv));
+ }
+ assertArrayEquals(msg, values, values2);
+ // test decoding to int[]
+ if (bpv <= 32) {
+ final int[] intValues2 = new int[values2.length];
+ decoder.decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, iterations);
+ assertTrue(msg, equals(intValues2, values2));
+ }
+
+ // 5. byte[] encoding
+ final byte[] blocks3 = new byte[8 * (blocksOffset2 + blocksLen)];
+ encoder.encode(values, valuesOffset, blocks3, 8 * blocksOffset2, iterations);
+ assertEquals(msg, LongBuffer.wrap(blocks2), ByteBuffer.wrap(blocks3).asLongBuffer());
+ // test encoding from int[]
+ if (bpv <= 32) {
+ final byte[] blocks4 = new byte[blocks3.length];
+ encoder.encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, iterations);
+ assertArrayEquals(msg, blocks3, blocks4);
+ }
+ }
+ }
+ }
+
+ private static boolean equals(int[] ints, long[] longs) {
+ if (ints.length != longs.length) {
+ return false;
+ }
+ for (int i = 0; i < ints.length; ++i) {
+ if ((ints[i] & 0xFFFFFFFFL) != longs[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
}
Modified: lucene/dev/branches/lucene3312/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java (original)
+++ lucene/dev/branches/lucene3312/lucene/facet/src/test/org/apache/lucene/util/SlowRAMDirectory.java Sat Aug 25 10:06:07 2012
@@ -107,7 +107,7 @@ public class SlowRAMDirectory extends RA
ii.readBytes(b, offset, len);
}
- @Override public IndexInput clone() { return (IndexInput)ii.clone(); }
+ @Override public IndexInput clone() { return ii.clone(); }
@Override public void close() throws IOException { ii.close(); }
@Override public boolean equals(Object o) { return ii.equals(o); }
@Override public long getFilePointer() { return ii.getFilePointer(); }
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java Sat Aug 25 10:06:07 2012
@@ -33,6 +33,9 @@ import org.apache.lucene.index.TermsEnum
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
+/**
+ * TokenStream created from a term vector field.
+ */
public final class TokenStreamFromTermPositionVector extends TokenStream {
private final List<Token> positionedTokens = new ArrayList<Token>();
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BaseFragmentsBuilder.java Sat Aug 25 10:06:07 2012
@@ -38,6 +38,12 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+/**
+ * Base FragmentsBuilder implementation that supports colored pre/post
+ * tags and multivalued fields.
+ * <p>
+ * Uses {@link BoundaryScanner} to determine fragments.
+ */
public abstract class BaseFragmentsBuilder implements FragmentsBuilder {
protected String[] preTags, postTags;
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/BoundaryScanner.java Sat Aug 25 10:06:07 2012
@@ -18,7 +18,7 @@ package org.apache.lucene.search.vectorh
*/
/**
- *
+ * Finds fragment boundaries: pluggable into {@link BaseFragmentsBuilder}
*/
public interface BoundaryScanner {
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldFragList.java Sat Aug 25 10:06:07 2012
@@ -57,6 +57,9 @@ public abstract class FieldFragList {
return fragInfos;
}
+ /**
+ * List of term offsets + weight for a frag info
+ */
public static class WeightedFragInfo {
private List<SubInfo> subInfos;
@@ -97,6 +100,9 @@ public abstract class FieldFragList {
return sb.toString();
}
+ /**
+ * Represents the list of term offsets for some text
+ */
public static class SubInfo {
private final String text; // unnecessary member, just exists for debugging purpose
private final List<Toffs> termsOffsets; // usually termsOffsets.size() == 1,
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldPhraseList.java Sat Aug 25 10:06:07 2012
@@ -118,6 +118,9 @@ public class FieldPhraseList {
getPhraseList().add( wpi );
}
+ /**
+ * Represents the list of term offsets and boost for some text
+ */
public static class WeightedPhraseInfo {
private String text; // unnecessary member, just exists for debugging purpose
@@ -230,6 +233,9 @@ public class FieldPhraseList {
return seqnum;
}
+ /**
+ * Term offsets (start + end)
+ */
public static class Toffs {
private int startOffset;
private int endOffset;
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldQuery.java Sat Aug 25 10:06:07 2012
@@ -321,6 +321,10 @@ public class FieldQuery {
return termOrPhraseNumber++;
}
+ /**
+ * Internal structure of a query for highlighting: represents
+ * a nested query structure
+ */
public static class QueryPhraseMap {
boolean terminal;
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/FieldTermStack.java Sat Aug 25 10:06:07 2012
@@ -159,6 +159,9 @@ public class FieldTermStack {
return termList == null || termList.size() == 0;
}
+ /**
+ * Single term with its position/offsets in the document and IDF weight
+ */
public static class TermInfo implements Comparable<TermInfo>{
private final String text;
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/ScoreOrderFragmentsBuilder.java Sat Aug 25 10:06:07 2012
@@ -62,6 +62,10 @@ public class ScoreOrderFragmentsBuilder
return src;
}
+ /**
+ * Comparator for {@link WeightedFragInfo} by boost, breaking ties
+ * by offset.
+ */
public static class ScoreComparator implements Comparator<WeightedFragInfo> {
public int compare( WeightedFragInfo o1, WeightedFragInfo o2 ) {
Modified: lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java?rev=1377246&r1=1377245&r2=1377246&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java (original)
+++ lucene/dev/branches/lucene3312/lucene/highlighter/src/java/org/apache/lucene/search/vectorhighlight/SimpleBoundaryScanner.java Sat Aug 25 10:06:07 2012
@@ -21,6 +21,10 @@ import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+/**
+ * Simple boundary scanner implementation that divides fragments
+ * based on a set of separator characters.
+ */
public class SimpleBoundaryScanner implements BoundaryScanner {
public static final int DEFAULT_MAX_SCAN = 20;