You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/01/29 16:29:44 UTC
svn commit: r1439938 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/lucene41/
lucene/core/src/java/org/apache/lucene/util/packed/
lucene/core/src/test/org/apache/lucene/util/packed/
Author: jpountz
Date: Tue Jan 29 15:29:42 2013
New Revision: 1439938
URL: http://svn.apache.org/viewvc?rev=1439938&view=rev
Log:
LUCENE-4726: Make PackedInts encoding from/to byte byte-aligned (merged from r1439886).
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked10.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked11.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked12.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked13.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked14.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked15.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked16.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked17.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked18.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked19.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked2.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked20.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked21.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked22.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked23.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked24.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked3.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked4.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked5.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked6.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked7.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked8.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked9.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPackedSingleBlock.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/ForUtil.java Tue Jan 29 15:29:42 2013
@@ -63,7 +63,7 @@ final class ForUtil {
}
final PackedInts.Decoder decoder = PackedInts.getDecoder(format, version, bpv);
final int iterations = computeIterations(decoder);
- maxDataSize = Math.max(maxDataSize, iterations * decoder.valueCount());
+ maxDataSize = Math.max(maxDataSize, iterations * decoder.byteValueCount());
}
}
}
@@ -75,7 +75,7 @@ final class ForUtil {
* values with the provided {@link Decoder}.
*/
private static int computeIterations(PackedInts.Decoder decoder) {
- return (int) Math.ceil((float) BLOCK_SIZE / decoder.valueCount());
+ return (int) Math.ceil((float) BLOCK_SIZE / decoder.byteValueCount());
}
/**
@@ -165,9 +165,9 @@ final class ForUtil {
assert numBits > 0 && numBits <= 32 : numBits;
final PackedInts.Encoder encoder = encoders[numBits];
final int iters = iterations[numBits];
- assert iters * encoder.valueCount() >= BLOCK_SIZE;
+ assert iters * encoder.byteValueCount() >= BLOCK_SIZE;
final int encodedSize = encodedSizes[numBits];
- assert (iters * encoder.blockCount()) << 3 >= encodedSize;
+ assert iters * encoder.byteBlockCount() >= encodedSize;
out.writeByte((byte) numBits);
@@ -198,7 +198,7 @@ final class ForUtil {
final PackedInts.Decoder decoder = decoders[numBits];
final int iters = iterations[numBits];
- assert iters * decoder.valueCount() >= BLOCK_SIZE;
+ assert iters * decoder.byteValueCount() >= BLOCK_SIZE;
decoder.decode(encoded, 0, decoded, 0, iters);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReader.java Tue Jan 29 15:29:42 2013
@@ -212,8 +212,8 @@ public final class BlockPackedReader {
Arrays.fill(values, minValue);
} else {
final PackedInts.Decoder decoder = PackedInts.getDecoder(PackedInts.Format.PACKED, packedIntsVersion, bitsPerValue);
- final int iterations = blockSize / decoder.valueCount();
- final int blocksSize = iterations * 8 * decoder.blockCount();
+ final int iterations = blockSize / decoder.byteValueCount();
+ final int blocksSize = iterations * decoder.byteBlockCount();
if (blocks == null || blocks.length < blocksSize) {
blocks = new byte[blocksSize];
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java Tue Jan 29 15:29:42 2013
@@ -147,8 +147,8 @@ public final class BlockPackedWriter {
}
}
final PackedInts.Encoder encoder = PackedInts.getEncoder(PackedInts.Format.PACKED, PackedInts.VERSION_CURRENT, bitsRequired);
- final int iterations = values.length / encoder.valueCount();
- final int blockSize = encoder.blockCount() * 8 * iterations;
+ final int iterations = values.length / encoder.byteValueCount();
+ final int blockSize = encoder.byteBlockCount() * iterations;
if (blocks == null || blocks.length < blockSize) {
blocks = new byte[blockSize];
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java Tue Jan 29 15:29:42 2013
@@ -2,7 +2,6 @@
package org.apache.lucene.util.packed;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -153,33 +152,28 @@ abstract class BulkOperation implements
* For every number of bits per value, there is a minimum number of
* blocks (b) / values (v) you need to write in order to reach the next block
* boundary:
- * - 16 bits per value -> b=1, v=4
- * - 24 bits per value -> b=3, v=8
- * - 50 bits per value -> b=25, v=32
- * - 63 bits per value -> b=63, v=64
+ * - 16 bits per value -> b=2, v=1
+ * - 24 bits per value -> b=3, v=1
+ * - 50 bits per value -> b=25, v=4
+ * - 63 bits per value -> b=63, v=8
* - ...
- * <p>
+ *
* A bulk read consists in copying <code>iterations*v</code> values that are
* contained in <code>iterations*b</code> blocks into a <code>long[]</code>
* (higher values of <code>iterations</code> are likely to yield a better
- * throughput) => this requires n * (b + v) longs in memory.
- * <p>
+ * throughput) => this requires n * (b + 8v) bytes of memory.
+ *
* This method computes <code>iterations</code> as
- * <code>ramBudget / (8 * (b + v))</code> (since a long is 8 bytes).
- * <p>
- * The resulting number of iterations of this method is guaranteed not to
- * overflow when multiplied by
- * <tt>8 * {@link PackedInts.Encoder#blockCount()}</tt> or
- * <tt>8 * {@link PackedInts.Decoder#blockCount()}</tt>.
+ * <code>ramBudget / (b + 8v)</code> (since a long is 8 bytes).
*/
public final int computeIterations(int valueCount, int ramBudget) {
- final int iterations = (ramBudget >>> 3) / (blockCount() + valueCount());
+ final int iterations = ramBudget / (byteBlockCount() + 8 * byteValueCount());
if (iterations == 0) {
// at least 1
return 1;
- } else if ((iterations - 1) * blockCount() >= valueCount) {
+ } else if ((iterations - 1) * byteValueCount() >= valueCount) {
// don't allocate for more than the size of the reader
- return (int) Math.ceil((double) valueCount / valueCount());
+ return (int) Math.ceil((double) valueCount / byteValueCount());
} else {
return iterations;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java Tue Jan 29 15:29:42 2013
@@ -1,5 +1,6 @@
package org.apache.lucene.util.packed;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -23,9 +24,12 @@ package org.apache.lucene.util.packed;
class BulkOperationPacked extends BulkOperation {
private final int bitsPerValue;
- private final int blockCount;
- private final int valueCount;
+ private final int longBlockCount;
+ private final int longValueCount;
+ private final int byteBlockCount;
+ private final int byteValueCount;
private final long mask;
+ private final int intMask;
public BulkOperationPacked(int bitsPerValue) {
this.bitsPerValue = bitsPerValue;
@@ -34,31 +38,50 @@ class BulkOperationPacked extends BulkOp
while ((blocks & 1) == 0) {
blocks >>>= 1;
}
- this.blockCount = blocks;
- this.valueCount = 64 * blockCount / bitsPerValue;
+ this.longBlockCount = blocks;
+ this.longValueCount = 64 * longBlockCount / bitsPerValue;
+ int byteBlockCount = 8 * longBlockCount;
+ int byteValueCount = longValueCount;
+ while ((byteBlockCount & 1) == 0 && (byteValueCount & 1) == 0) {
+ byteBlockCount >>>= 1;
+ byteValueCount >>>= 1;
+ }
+ this.byteBlockCount = byteBlockCount;
+ this.byteValueCount = byteValueCount;
if (bitsPerValue == 64) {
this.mask = ~0L;
} else {
this.mask = (1L << bitsPerValue) - 1;
}
- assert valueCount * bitsPerValue == 64 * blockCount;
+ this.intMask = (int) mask;
+ assert longValueCount * bitsPerValue == 64 * longBlockCount;
+ }
+
+ @Override
+ public int longBlockCount() {
+ return longBlockCount;
}
@Override
- public int blockCount() {
- return blockCount;
+ public int longValueCount() {
+ return longValueCount;
}
@Override
- public int valueCount() {
- return valueCount;
+ public int byteBlockCount() {
+ return byteBlockCount;
+ }
+
+ @Override
+ public int byteValueCount() {
+ return byteValueCount;
}
@Override
public void decode(long[] blocks, int blocksOffset, long[] values,
int valuesOffset, int iterations) {
int bitsLeft = 64;
- for (int i = 0; i < valueCount * iterations; ++i) {
+ for (int i = 0; i < longValueCount * iterations; ++i) {
bitsLeft -= bitsPerValue;
if (bitsLeft < 0) {
values[valuesOffset++] =
@@ -74,22 +97,28 @@ class BulkOperationPacked extends BulkOp
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values,
int valuesOffset, int iterations) {
- int blockBitsLeft = 8;
- int valueBitsLeft = bitsPerValue;
- long nextValue = 0;
- for (int end = valuesOffset + iterations * valueCount; valuesOffset < end; ) {
- if (valueBitsLeft > blockBitsLeft) {
- nextValue |= (blocks[blocksOffset++] & ((1L << blockBitsLeft) - 1)) << (valueBitsLeft - blockBitsLeft);
- valueBitsLeft -= blockBitsLeft;
- blockBitsLeft = 8;
- } else {
- nextValue |= ((blocks[blocksOffset] & 0xFFL) >>> (blockBitsLeft - valueBitsLeft)) & ((1L << valueBitsLeft) - 1);
- values[valuesOffset++] = nextValue;
- nextValue = 0;
- blockBitsLeft -= valueBitsLeft;
- valueBitsLeft = bitsPerValue;
+ long nextValue = 0L;
+ int bitsLeft = bitsPerValue;
+ for (int i = 0; i < iterations * byteBlockCount; ++i) {
+ final long bytes = blocks[blocksOffset++] & 0xFFL;
+ if (bitsLeft > 8) {
+ // just buffer
+ bitsLeft -= 8;
+ nextValue |= bytes << bitsLeft;
+ } else {
+ // flush
+ int bits = 8 - bitsLeft;
+ values[valuesOffset++] = nextValue | (bytes >>> bits);
+ while (bits >= bitsPerValue) {
+ bits -= bitsPerValue;
+ values[valuesOffset++] = (bytes >>> bits) & mask;
+ }
+ // then buffer
+ bitsLeft = bitsPerValue - bits;
+ nextValue = (bytes & ((1L << bits) - 1)) << bitsLeft;
}
}
+ assert bitsLeft == bitsPerValue;
}
@Override
@@ -99,7 +128,7 @@ class BulkOperationPacked extends BulkOp
throw new UnsupportedOperationException("Cannot decode " + bitsPerValue + "-bits values into an int[]");
}
int bitsLeft = 64;
- for (int i = 0; i < valueCount * iterations; ++i) {
+ for (int i = 0; i < longValueCount * iterations; ++i) {
bitsLeft -= bitsPerValue;
if (bitsLeft < 0) {
values[valuesOffset++] = (int)
@@ -115,25 +144,28 @@ class BulkOperationPacked extends BulkOp
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values,
int valuesOffset, int iterations) {
- if (bitsPerValue > 32) {
- throw new UnsupportedOperationException("Cannot decode " + bitsPerValue + "-bits values into an int[]");
- }
- int blockBitsLeft = 8;
- int valueBitsLeft = bitsPerValue;
int nextValue = 0;
- for (int end = valuesOffset + iterations * valueCount; valuesOffset < end; ) {
- if (valueBitsLeft > blockBitsLeft) {
- nextValue |= (blocks[blocksOffset++] & ((1L << blockBitsLeft) - 1)) << (valueBitsLeft - blockBitsLeft);
- valueBitsLeft -= blockBitsLeft;
- blockBitsLeft = 8;
- } else {
- nextValue |= ((blocks[blocksOffset] & 0xFFL) >>> (blockBitsLeft - valueBitsLeft)) & ((1L << valueBitsLeft) - 1);
- values[valuesOffset++] = nextValue;
- nextValue = 0;
- blockBitsLeft -= valueBitsLeft;
- valueBitsLeft = bitsPerValue;
+ int bitsLeft = bitsPerValue;
+ for (int i = 0; i < iterations * byteBlockCount; ++i) {
+ final int bytes = blocks[blocksOffset++] & 0xFF;
+ if (bitsLeft > 8) {
+ // just buffer
+ bitsLeft -= 8;
+ nextValue |= bytes << bitsLeft;
+ } else {
+ // flush
+ int bits = 8 - bitsLeft;
+ values[valuesOffset++] = nextValue | (bytes >>> bits);
+ while (bits >= bitsPerValue) {
+ bits -= bitsPerValue;
+ values[valuesOffset++] = (bytes >>> bits) & intMask;
+ }
+ // then buffer
+ bitsLeft = bitsPerValue - bits;
+ nextValue = (bytes & ((1 << bits) - 1)) << bitsLeft;
}
}
+ assert bitsLeft == bitsPerValue;
}
@Override
@@ -141,7 +173,7 @@ class BulkOperationPacked extends BulkOp
int blocksOffset, int iterations) {
long nextBlock = 0;
int bitsLeft = 64;
- for (int i = 0; i < valueCount * iterations; ++i) {
+ for (int i = 0; i < longValueCount * iterations; ++i) {
bitsLeft -= bitsPerValue;
if (bitsLeft > 0) {
nextBlock |= values[valuesOffset++] << bitsLeft;
@@ -164,7 +196,7 @@ class BulkOperationPacked extends BulkOp
int blocksOffset, int iterations) {
long nextBlock = 0;
int bitsLeft = 64;
- for (int i = 0; i < valueCount * iterations; ++i) {
+ for (int i = 0; i < longValueCount * iterations; ++i) {
bitsLeft -= bitsPerValue;
if (bitsLeft > 0) {
nextBlock |= (values[valuesOffset++] & 0xFFFFFFFFL) << bitsLeft;
@@ -185,47 +217,57 @@ class BulkOperationPacked extends BulkOp
@Override
public void encode(long[] values, int valuesOffset, byte[] blocks,
int blocksOffset, int iterations) {
- long nextBlock = 0;
- int bitsLeft = 64;
- for (int i = 0; i < valueCount * iterations; ++i) {
- bitsLeft -= bitsPerValue;
- if (bitsLeft > 0) {
- nextBlock |= values[valuesOffset++] << bitsLeft;
- } else if (bitsLeft == 0) {
- nextBlock |= values[valuesOffset++];
- blocksOffset = writeLong(nextBlock, blocks, blocksOffset);
- nextBlock = 0;
- bitsLeft = 64;
- } else { // bitsLeft < 0
- nextBlock |= values[valuesOffset] >>> -bitsLeft;
- blocksOffset = writeLong(nextBlock, blocks, blocksOffset);
- nextBlock = (values[valuesOffset++] & ((1L << -bitsLeft) - 1)) << (64 + bitsLeft);
- bitsLeft += 64;
+ int nextBlock = 0;
+ int bitsLeft = 8;
+ for (int i = 0; i < byteValueCount * iterations; ++i) {
+ final long v = values[valuesOffset++];
+ assert bitsPerValue == 64 || PackedInts.bitsRequired(v) <= bitsPerValue;
+ if (bitsPerValue < bitsLeft) {
+ // just buffer
+ nextBlock |= v << (bitsLeft - bitsPerValue);
+ bitsLeft -= bitsPerValue;
+ } else {
+ // flush as many blocks as possible
+ int bits = bitsPerValue - bitsLeft;
+ blocks[blocksOffset++] = (byte) (nextBlock | (v >>> bits));
+ while (bits >= 8) {
+ bits -= 8;
+ blocks[blocksOffset++] = (byte) (v >>> bits);
+ }
+ // then buffer
+ bitsLeft = 8 - bits;
+ nextBlock = (int) ((v & ((1L << bits) - 1)) << bitsLeft);
}
}
+ assert bitsLeft == 8;
}
@Override
public void encode(int[] values, int valuesOffset, byte[] blocks,
int blocksOffset, int iterations) {
- long nextBlock = 0;
- int bitsLeft = 64;
- for (int i = 0; i < valueCount * iterations; ++i) {
- bitsLeft -= bitsPerValue;
- if (bitsLeft > 0) {
- nextBlock |= (values[valuesOffset++] & 0xFFFFFFFFL) << bitsLeft;
- } else if (bitsLeft == 0) {
- nextBlock |= (values[valuesOffset++] & 0xFFFFFFFFL);
- blocksOffset = writeLong(nextBlock, blocks, blocksOffset);
- nextBlock = 0;
- bitsLeft = 64;
- } else { // bitsLeft < 0
- nextBlock |= (values[valuesOffset] & 0xFFFFFFFFL) >>> -bitsLeft;
- blocksOffset = writeLong(nextBlock, blocks, blocksOffset);
- nextBlock = (values[valuesOffset++] & ((1L << -bitsLeft) - 1)) << (64 + bitsLeft);
- bitsLeft += 64;
+ int nextBlock = 0;
+ int bitsLeft = 8;
+ for (int i = 0; i < byteValueCount * iterations; ++i) {
+ final int v = values[valuesOffset++];
+ assert PackedInts.bitsRequired(v & 0xFFFFFFFFL) <= bitsPerValue;
+ if (bitsPerValue < bitsLeft) {
+ // just buffer
+ nextBlock |= v << (bitsLeft - bitsPerValue);
+ bitsLeft -= bitsPerValue;
+ } else {
+ // flush as many blocks as possible
+ int bits = bitsPerValue - bitsLeft;
+ blocks[blocksOffset++] = (byte) (nextBlock | (v >>> bits));
+ while (bits >= 8) {
+ bits -= 8;
+ blocks[blocksOffset++] = (byte) (v >>> bits);
+ }
+ // then buffer
+ bitsLeft = 8 - bits;
+ nextBlock = (v & ((1 << bits) - 1)) << bitsLeft;
}
}
+ assert bitsLeft == 8;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked1.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked1 extends
public BulkOperationPacked1() {
super(1);
- assert blockCount() == 1;
- assert valueCount() == 64;
}
@Override
@@ -42,7 +40,7 @@ final class BulkOperationPacked1 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
final byte block = blocks[blocksOffset++];
values[valuesOffset++] = (block >>> 7) & 1;
values[valuesOffset++] = (block >>> 6) & 1;
@@ -67,7 +65,7 @@ final class BulkOperationPacked1 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
final byte block = blocks[blocksOffset++];
values[valuesOffset++] = (block >>> 7) & 1;
values[valuesOffset++] = (block >>> 6) & 1;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked10.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked10.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked10.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked10.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked10 extend
public BulkOperationPacked10() {
super(10);
- assert blockCount() == 5;
- assert valueCount() == 32;
}
@Override
@@ -75,7 +73,7 @@ final class BulkOperationPacked10 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 2) | (byte1 >>> 6);
@@ -133,7 +131,7 @@ final class BulkOperationPacked10 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 2) | (byte1 >>> 6);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked11.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked11.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked11.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked11.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked11 extend
public BulkOperationPacked11() {
super(11);
- assert blockCount() == 11;
- assert valueCount() == 64;
}
@Override
@@ -113,7 +111,7 @@ final class BulkOperationPacked11 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 3) | (byte1 >>> 5);
@@ -219,7 +217,7 @@ final class BulkOperationPacked11 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 3) | (byte1 >>> 5);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked12.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked12.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked12.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked12.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked12 extend
public BulkOperationPacked12() {
super(12);
- assert blockCount() == 3;
- assert valueCount() == 16;
}
@Override
@@ -57,7 +55,7 @@ final class BulkOperationPacked12 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 4) | (byte1 >>> 4);
@@ -93,7 +91,7 @@ final class BulkOperationPacked12 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 4) | (byte1 >>> 4);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked13.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked13.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked13.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked13.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked13 extend
public BulkOperationPacked13() {
super(13);
- assert blockCount() == 13;
- assert valueCount() == 64;
}
@Override
@@ -115,7 +113,7 @@ final class BulkOperationPacked13 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 5) | (byte1 >>> 3);
@@ -225,7 +223,7 @@ final class BulkOperationPacked13 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 5) | (byte1 >>> 3);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked14.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked14.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked14.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked14.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked14 extend
public BulkOperationPacked14() {
super(14);
- assert blockCount() == 7;
- assert valueCount() == 32;
}
@Override
@@ -77,7 +75,7 @@ final class BulkOperationPacked14 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 6) | (byte1 >>> 2);
@@ -139,7 +137,7 @@ final class BulkOperationPacked14 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 6) | (byte1 >>> 2);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked15.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked15.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked15.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked15.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked15 extend
public BulkOperationPacked15() {
super(15);
- assert blockCount() == 15;
- assert valueCount() == 64;
}
@Override
@@ -117,7 +115,7 @@ final class BulkOperationPacked15 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 7) | (byte1 >>> 1);
@@ -231,7 +229,7 @@ final class BulkOperationPacked15 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 7) | (byte1 >>> 1);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked16.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked16.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked16.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked16.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked16 extend
public BulkOperationPacked16() {
super(16);
- assert blockCount() == 1;
- assert valueCount() == 4;
}
@Override
@@ -42,7 +40,7 @@ final class BulkOperationPacked16 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 4 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
values[valuesOffset++] = ((blocks[blocksOffset++] & 0xFF) << 8) | (blocks[blocksOffset++] & 0xFF);
}
}
@@ -59,7 +57,7 @@ final class BulkOperationPacked16 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 4 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
values[valuesOffset++] = ((blocks[blocksOffset++] & 0xFFL) << 8) | (blocks[blocksOffset++] & 0xFFL);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked17.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked17.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked17.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked17.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked17 extend
public BulkOperationPacked17() {
super(17);
- assert blockCount() == 17;
- assert valueCount() == 64;
}
@Override
@@ -119,7 +117,7 @@ final class BulkOperationPacked17 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -237,7 +235,7 @@ final class BulkOperationPacked17 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked18.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked18.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked18.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked18.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked18 extend
public BulkOperationPacked18() {
super(18);
- assert blockCount() == 9;
- assert valueCount() == 32;
}
@Override
@@ -79,7 +77,7 @@ final class BulkOperationPacked18 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -145,7 +143,7 @@ final class BulkOperationPacked18 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked19.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked19.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked19.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked19.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked19 extend
public BulkOperationPacked19() {
super(19);
- assert blockCount() == 19;
- assert valueCount() == 64;
}
@Override
@@ -121,7 +119,7 @@ final class BulkOperationPacked19 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -243,7 +241,7 @@ final class BulkOperationPacked19 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked2.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked2.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked2.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked2 extends
public BulkOperationPacked2() {
super(2);
- assert blockCount() == 1;
- assert valueCount() == 32;
}
@Override
@@ -42,7 +40,7 @@ final class BulkOperationPacked2 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
final byte block = blocks[blocksOffset++];
values[valuesOffset++] = (block >>> 6) & 3;
values[valuesOffset++] = (block >>> 4) & 3;
@@ -63,7 +61,7 @@ final class BulkOperationPacked2 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
final byte block = blocks[blocksOffset++];
values[valuesOffset++] = (block >>> 6) & 3;
values[valuesOffset++] = (block >>> 4) & 3;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked20.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked20.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked20.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked20.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked20 extend
public BulkOperationPacked20() {
super(20);
- assert blockCount() == 5;
- assert valueCount() == 16;
}
@Override
@@ -59,7 +57,7 @@ final class BulkOperationPacked20 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -99,7 +97,7 @@ final class BulkOperationPacked20 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked21.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked21.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked21.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked21.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked21 extend
public BulkOperationPacked21() {
super(21);
- assert blockCount() == 21;
- assert valueCount() == 64;
}
@Override
@@ -123,7 +121,7 @@ final class BulkOperationPacked21 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -249,7 +247,7 @@ final class BulkOperationPacked21 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked22.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked22.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked22.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked22.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked22 extend
public BulkOperationPacked22() {
super(22);
- assert blockCount() == 11;
- assert valueCount() == 32;
}
@Override
@@ -81,7 +79,7 @@ final class BulkOperationPacked22 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -151,7 +149,7 @@ final class BulkOperationPacked22 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked23.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked23.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked23.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked23.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked23 extend
public BulkOperationPacked23() {
super(23);
- assert blockCount() == 23;
- assert valueCount() == 64;
}
@Override
@@ -125,7 +123,7 @@ final class BulkOperationPacked23 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -255,7 +253,7 @@ final class BulkOperationPacked23 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked24.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked24.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked24.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked24.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked24 extend
public BulkOperationPacked24() {
super(24);
- assert blockCount() == 3;
- assert valueCount() == 8;
}
@Override
@@ -49,7 +47,7 @@ final class BulkOperationPacked24 extend
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
final int byte2 = blocks[blocksOffset++] & 0xFF;
@@ -76,7 +74,7 @@ final class BulkOperationPacked24 extend
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
final long byte2 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked3.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked3.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked3.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked3.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked3 extends
public BulkOperationPacked3() {
super(3);
- assert blockCount() == 3;
- assert valueCount() == 64;
}
@Override
@@ -105,7 +103,7 @@ final class BulkOperationPacked3 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 5;
values[valuesOffset++] = (byte0 >>> 2) & 7;
@@ -195,7 +193,7 @@ final class BulkOperationPacked3 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 5;
values[valuesOffset++] = (byte0 >>> 2) & 7;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked4.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked4.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked4.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked4.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked4 extends
public BulkOperationPacked4() {
super(4);
- assert blockCount() == 1;
- assert valueCount() == 16;
}
@Override
@@ -42,7 +40,7 @@ final class BulkOperationPacked4 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
final byte block = blocks[blocksOffset++];
values[valuesOffset++] = (block >>> 4) & 15;
values[valuesOffset++] = block & 15;
@@ -61,7 +59,7 @@ final class BulkOperationPacked4 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
final byte block = blocks[blocksOffset++];
values[valuesOffset++] = (block >>> 4) & 15;
values[valuesOffset++] = block & 15;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked5.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked5.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked5.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked5.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked5 extends
public BulkOperationPacked5() {
super(5);
- assert blockCount() == 5;
- assert valueCount() == 64;
}
@Override
@@ -107,7 +105,7 @@ final class BulkOperationPacked5 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 3;
final int byte1 = blocks[blocksOffset++] & 0xFF;
@@ -201,7 +199,7 @@ final class BulkOperationPacked5 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 3;
final long byte1 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked6.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked6.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked6.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked6.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked6 extends
public BulkOperationPacked6() {
super(6);
- assert blockCount() == 3;
- assert valueCount() == 32;
}
@Override
@@ -73,7 +71,7 @@ final class BulkOperationPacked6 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 2;
final int byte1 = blocks[blocksOffset++] & 0xFF;
@@ -127,7 +125,7 @@ final class BulkOperationPacked6 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 2;
final long byte1 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked7.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked7.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked7.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked7.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked7 extends
public BulkOperationPacked7() {
super(7);
- assert blockCount() == 7;
- assert valueCount() == 64;
}
@Override
@@ -109,7 +107,7 @@ final class BulkOperationPacked7 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 1;
final int byte1 = blocks[blocksOffset++] & 0xFF;
@@ -207,7 +205,7 @@ final class BulkOperationPacked7 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = byte0 >>> 1;
final long byte1 = blocks[blocksOffset++] & 0xFF;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked8.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked8.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked8.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked8 extends
public BulkOperationPacked8() {
super(8);
- assert blockCount() == 1;
- assert valueCount() == 8;
}
@Override
@@ -42,7 +40,7 @@ final class BulkOperationPacked8 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
values[valuesOffset++] = blocks[blocksOffset++] & 0xFF;
}
}
@@ -59,7 +57,7 @@ final class BulkOperationPacked8 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int j = 0; j < 8 * iterations; ++j) {
+ for (int j = 0; j < iterations; ++j) {
values[valuesOffset++] = blocks[blocksOffset++] & 0xFF;
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked9.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked9.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked9.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked9.java Tue Jan 29 15:29:42 2013
@@ -26,8 +26,6 @@ final class BulkOperationPacked9 extends
public BulkOperationPacked9() {
super(9);
- assert blockCount() == 9;
- assert valueCount() == 64;
}
@Override
@@ -111,7 +109,7 @@ final class BulkOperationPacked9 extends
@Override
public void decode(byte[] blocks, int blocksOffset, int[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final int byte0 = blocks[blocksOffset++] & 0xFF;
final int byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 1) | (byte1 >>> 7);
@@ -213,7 +211,7 @@ final class BulkOperationPacked9 extends
@Override
public void decode(byte[] blocks, int blocksOffset, long[] values, int valuesOffset, int iterations) {
- for (int i = 0; i < 8 * iterations; ++i) {
+ for (int i = 0; i < iterations; ++i) {
final long byte0 = blocks[blocksOffset++] & 0xFF;
final long byte1 = blocks[blocksOffset++] & 0xFF;
values[valuesOffset++] = (byte0 << 1) | (byte1 >>> 7);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPackedSingleBlock.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPackedSingleBlock.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPackedSingleBlock.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPackedSingleBlock.java Tue Jan 29 15:29:42 2013
@@ -35,12 +35,22 @@ final class BulkOperationPackedSingleBlo
}
@Override
- public final int blockCount() {
+ public final int longBlockCount() {
return BLOCK_COUNT;
}
@Override
- public int valueCount() {
+ public final int byteBlockCount() {
+ return BLOCK_COUNT * 8;
+ }
+
+ @Override
+ public int longValueCount() {
+ return valueCount;
+ }
+
+ @Override
+ public final int byteValueCount() {
return valueCount;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java Tue Jan 29 15:29:42 2013
@@ -140,9 +140,9 @@ class Packed64 extends PackedInts.Mutabl
final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
// go to the next block where the value does not span across two blocks
- final int offsetInBlocks = index % decoder.valueCount();
+ final int offsetInBlocks = index % decoder.longValueCount();
if (offsetInBlocks != 0) {
- for (int i = offsetInBlocks; i < decoder.valueCount() && len > 0; ++i) {
+ for (int i = offsetInBlocks; i < decoder.longValueCount() && len > 0; ++i) {
arr[off++] = get(index++);
--len;
}
@@ -152,12 +152,12 @@ class Packed64 extends PackedInts.Mutabl
}
// bulk get
- assert index % decoder.valueCount() == 0;
+ assert index % decoder.longValueCount() == 0;
int blockIndex = (int) ((long) index * bitsPerValue) >>> BLOCK_BITS;
assert (((long)index * bitsPerValue) & MOD_MASK) == 0;
- final int iterations = len / decoder.valueCount();
+ final int iterations = len / decoder.longValueCount();
decoder.decode(blocks, blockIndex, arr, off, iterations);
- final int gotValues = iterations * decoder.valueCount();
+ final int gotValues = iterations * decoder.longValueCount();
index += gotValues;
len -= gotValues;
assert len >= 0;
@@ -204,9 +204,9 @@ class Packed64 extends PackedInts.Mutabl
final PackedInts.Encoder encoder = BulkOperation.of(PackedInts.Format.PACKED, bitsPerValue);
// go to the next block where the value does not span across two blocks
- final int offsetInBlocks = index % encoder.valueCount();
+ final int offsetInBlocks = index % encoder.longValueCount();
if (offsetInBlocks != 0) {
- for (int i = offsetInBlocks; i < encoder.valueCount() && len > 0; ++i) {
+ for (int i = offsetInBlocks; i < encoder.longValueCount() && len > 0; ++i) {
set(index++, arr[off++]);
--len;
}
@@ -216,12 +216,12 @@ class Packed64 extends PackedInts.Mutabl
}
// bulk set
- assert index % encoder.valueCount() == 0;
+ assert index % encoder.longValueCount() == 0;
int blockIndex = (int) ((long) index * bitsPerValue) >>> BLOCK_BITS;
assert (((long)index * bitsPerValue) & MOD_MASK) == 0;
- final int iterations = len / encoder.valueCount();
+ final int iterations = len / encoder.longValueCount();
encoder.encode(arr, off, blocks, blockIndex, iterations);
- final int setValues = iterations * encoder.valueCount();
+ final int setValues = iterations * encoder.longValueCount();
index += setValues;
len -= setValues;
assert len >= 0;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java Tue Jan 29 15:29:42 2013
@@ -88,8 +88,8 @@ abstract class Packed64SingleBlock exten
// bulk get
assert index % valuesPerBlock == 0;
final PackedInts.Decoder decoder = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
- assert decoder.blockCount() == 1;
- assert decoder.valueCount() == valuesPerBlock;
+ assert decoder.longBlockCount() == 1;
+ assert decoder.longValueCount() == valuesPerBlock;
final int blockIndex = index / valuesPerBlock;
final int nblocks = (index + len) / valuesPerBlock - blockIndex;
decoder.decode(blocks, blockIndex, arr, off, nblocks);
@@ -132,8 +132,8 @@ abstract class Packed64SingleBlock exten
// bulk set
assert index % valuesPerBlock == 0;
final BulkOperation op = BulkOperation.of(PackedInts.Format.PACKED_SINGLE_BLOCK, bitsPerValue);
- assert op.blockCount() == 1;
- assert op.valueCount() == valuesPerBlock;
+ assert op.longBlockCount() == 1;
+ assert op.longValueCount() == valuesPerBlock;
final int blockIndex = index / valuesPerBlock;
final int nblocks = (index + len) / valuesPerBlock - blockIndex;
op.encode(arr, off, blocks, blockIndex, nblocks);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Tue Jan 29 15:29:42 2013
@@ -280,15 +280,28 @@ public class PackedInts {
public static interface Decoder {
/**
- * The minimum number of long blocks to decode in a single call.
+ * The minimum number of long blocks to encode in a single iteration, when
+ * using long encoding.
*/
- int blockCount();
+ int longBlockCount();
/**
- * The number of values that can be stored in <code>blockCount()</code> long
+ * The number of values that can be stored in {@link #longBlockCount()} long
* blocks.
*/
- int valueCount();
+ int longValueCount();
+
+ /**
+ * The minimum number of byte blocks to encode in a single iteration, when
+ * using byte encoding.
+ */
+ int byteBlockCount();
+
+ /**
+ * The number of values that can be stored in {@link #byteBlockCount()} byte
+ * blocks.
+ */
+ int byteValueCount();
/**
* Read <code>iterations * blockCount()</code> blocks from <code>blocks</code>,
@@ -350,15 +363,28 @@ public class PackedInts {
public static interface Encoder {
/**
- * The minimum number of long blocks to encode in a single call.
+ * The minimum number of long blocks to encode in a single iteration, when
+ * using long encoding.
+ */
+ int longBlockCount();
+
+ /**
+ * The number of values that can be stored in {@link #longBlockCount()} long
+ * blocks.
+ */
+ int longValueCount();
+
+ /**
+ * The minimum number of byte blocks to encode in a single iteration, when
+ * using byte encoding.
*/
- int blockCount();
+ int byteBlockCount();
/**
- * The number of values that can be stored in <code>blockCount()</code> long
+ * The number of values that can be stored in {@link #byteBlockCount()} byte
* blocks.
*/
- int valueCount();
+ int byteValueCount();
/**
* Read <code>iterations * valueCount()</code> values from <code>values</code>,
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java Tue Jan 29 15:29:42 2013
@@ -39,14 +39,23 @@ final class PackedReaderIterator extends
this.format = format;
this.packedIntsVersion = packedIntsVersion;
bulkOperation = BulkOperation.of(format, bitsPerValue);
- iterations = bulkOperation.computeIterations(valueCount, mem);
+ iterations = iterations(mem);
assert valueCount == 0 || iterations > 0;
- nextBlocks = new byte[8 * iterations * bulkOperation.blockCount()];
- nextValues = new LongsRef(new long[iterations * bulkOperation.valueCount()], 0, 0);
+ nextBlocks = new byte[iterations * bulkOperation.byteBlockCount()];
+ nextValues = new LongsRef(new long[iterations * bulkOperation.byteValueCount()], 0, 0);
nextValues.offset = nextValues.longs.length;
position = -1;
}
+ private int iterations(int mem) {
+ int iterations = bulkOperation.computeIterations(valueCount, mem);
+ if (packedIntsVersion < PackedInts.VERSION_BYTE_ALIGNED) {
+ // make sure iterations is a multiple of 8
+ iterations = (iterations + 7) & 0xFFFFFFF8;
+ }
+ return iterations;
+ }
+
@Override
public LongsRef next(int count) throws IOException {
assert nextValues.length >= 0;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java Tue Jan 29 15:29:42 2013
@@ -42,8 +42,8 @@ final class PackedWriter extends PackedI
this.format = format;
encoder = BulkOperation.of(format, bitsPerValue);
iterations = encoder.computeIterations(valueCount, mem);
- nextBlocks = new byte[8 * iterations * encoder.blockCount()];
- nextValues = new long[iterations * encoder.valueCount()];
+ nextBlocks = new byte[iterations * encoder.byteBlockCount()];
+ nextValues = new long[iterations * encoder.byteValueCount()];
off = 0;
written = 0;
finished = false;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py Tue Jan 29 15:29:42 2013
@@ -57,28 +57,28 @@ FOOTER="""
* For every number of bits per value, there is a minimum number of
* blocks (b) / values (v) you need to write in order to reach the next block
* boundary:
- * - 16 bits per value -> b=1, v=4
- * - 24 bits per value -> b=3, v=8
- * - 50 bits per value -> b=25, v=32
- * - 63 bits per value -> b=63, v=64
+ * - 16 bits per value -> b=2, v=1
+ * - 24 bits per value -> b=3, v=1
+ * - 50 bits per value -> b=25, v=4
+ * - 63 bits per value -> b=63, v=8
* - ...
*
* A bulk read consists in copying <code>iterations*v</code> values that are
* contained in <code>iterations*b</code> blocks into a <code>long[]</code>
* (higher values of <code>iterations</code> are likely to yield a better
- * throughput) => this requires n * (b + v) longs in memory.
+ * throughput) => this requires n * (b + 8v) bytes of memory.
*
* This method computes <code>iterations</code> as
- * <code>ramBudget / (8 * (b + v))</code> (since a long is 8 bytes).
+ * <code>ramBudget / (b + 8v)</code> (since a long is 8 bytes).
*/
public final int computeIterations(int valueCount, int ramBudget) {
- final int iterations = (ramBudget >>> 3) / (blockCount() + valueCount());
+ final int iterations = ramBudget / (byteBlockCount() + 8 * byteValueCount());
if (iterations == 0) {
// at least 1
return 1;
- } else if ((iterations - 1) * blockCount() >= valueCount) {
+ } else if ((iterations - 1) * byteValueCount() >= valueCount) {
// don't allocate for more than the size of the reader
- return (int) Math.ceil((double) valueCount / valueCount());
+ return (int) Math.ceil((double) valueCount / byteValueCount());
} else {
return iterations;
}
@@ -131,14 +131,11 @@ def block_value_count(bpv, bits=64):
return (blocks, values)
def packed64(bpv, f):
- blocks, values = block_value_count(bpv)
mask = (1 << bpv) - 1
f.write("\n")
f.write(" public BulkOperationPacked%d() {\n" %bpv)
f.write(" super(%d);\n" %bpv)
- f.write(" assert blockCount() == %d;\n" %blocks)
- f.write(" assert valueCount() == %d;\n" %values)
f.write(" }\n\n")
if bpv == 64:
@@ -215,20 +212,19 @@ def p64_decode(bpv, f, bits):
if bits < bpv:
f.write(" throw new UnsupportedOperationException();\n")
else:
-
if is_power_of_two(bpv) and bpv < 8:
- f.write(" for (int j = 0; j < 8 * iterations; ++j) {\n")
+ f.write(" for (int j = 0; j < iterations; ++j) {\n")
f.write(" final byte block = blocks[blocksOffset++];\n")
for shift in xrange(8 - bpv, 0, -bpv):
f.write(" values[valuesOffset++] = (block >>> %d) & %d;\n" %(shift, mask))
f.write(" values[valuesOffset++] = block & %d;\n" %mask)
f.write(" }\n")
elif bpv == 8:
- f.write(" for (int j = 0; j < 8 * iterations; ++j) {\n")
+ f.write(" for (int j = 0; j < iterations; ++j) {\n")
f.write(" values[valuesOffset++] = blocks[blocksOffset++] & 0xFF;\n")
f.write(" }\n")
elif is_power_of_two(bpv) and bpv > 8:
- f.write(" for (int j = 0; j < %d * iterations; ++j) {\n" %(64 / bpv))
+ f.write(" for (int j = 0; j < iterations; ++j) {\n")
m = bits <= 32 and "0xFF" or "0xFFL"
f.write(" values[valuesOffset++] =")
for i in xrange(bpv / 8 - 1):
@@ -236,7 +232,7 @@ def p64_decode(bpv, f, bits):
f.write(" (blocks[blocksOffset++] & %s);\n" %m)
f.write(" }\n")
else:
- f.write(" for (int i = 0; i < 8 * iterations; ++i) {\n")
+ f.write(" for (int i = 0; i < iterations; ++i) {\n")
for i in xrange(0, byte_values):
byte_start = i * bpv / 8
bit_start = (i * bpv) % 8
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1439938&r1=1439937&r2=1439938&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Tue Jan 29 15:29:42 2013
@@ -203,7 +203,7 @@ public class TestPackedInts extends Luce
if (!format.isSupported(bpv)) {
continue;
}
- final long byteCount = format.byteCount(version, valueCount, bpv);
+ final long byteCount = format.byteCount(version, valueCount, bpv);
String msg = "format=" + format + ",version=" + version + ",valueCount=" + valueCount + ",bpv=" + bpv;
// test iterator
@@ -697,16 +697,22 @@ public class TestPackedInts extends Luce
final PackedInts.Encoder encoder = PackedInts.getEncoder(format, PackedInts.VERSION_CURRENT, bpv);
final PackedInts.Decoder decoder = PackedInts.getDecoder(format, PackedInts.VERSION_CURRENT, bpv);
- final int blockCount = encoder.blockCount();
- final int valueCount = encoder.valueCount();
- assertEquals(blockCount, decoder.blockCount());
- assertEquals(valueCount, decoder.valueCount());
-
- final int iterations = random().nextInt(100);
+ final int longBlockCount = encoder.longBlockCount();
+ final int longValueCount = encoder.longValueCount();
+ final int byteBlockCount = encoder.byteBlockCount();
+ final int byteValueCount = encoder.byteValueCount();
+ assertEquals(longBlockCount, decoder.longBlockCount());
+ assertEquals(longValueCount, decoder.longValueCount());
+ assertEquals(byteBlockCount, decoder.byteBlockCount());
+ assertEquals(byteValueCount, decoder.byteValueCount());
+
+ final int longIterations = random().nextInt(100);
+ final int byteIterations = longIterations * longValueCount / byteValueCount;
+ assertEquals(longIterations * longValueCount, byteIterations * byteValueCount);
final int blocksOffset = random().nextInt(100);
final int valuesOffset = random().nextInt(100);
final int blocksOffset2 = random().nextInt(100);
- final int blocksLen = iterations * blockCount;
+ final int blocksLen = longIterations * longBlockCount;
// 1. generate random inputs
final long[] blocks = new long[blocksOffset + blocksLen];
@@ -720,8 +726,8 @@ public class TestPackedInts extends Luce
}
// 2. decode
- final long[] values = new long[valuesOffset + iterations * valueCount];
- decoder.decode(blocks, blocksOffset, values, valuesOffset, iterations);
+ final long[] values = new long[valuesOffset + longIterations * longValueCount];
+ decoder.decode(blocks, blocksOffset, values, valuesOffset, longIterations);
for (long value : values) {
assertTrue(value <= PackedInts.maxValue(bpv));
}
@@ -729,7 +735,7 @@ public class TestPackedInts extends Luce
final int[] intValues;
if (bpv <= 32) {
intValues = new int[values.length];
- decoder.decode(blocks, blocksOffset, intValues, valuesOffset, iterations);
+ decoder.decode(blocks, blocksOffset, intValues, valuesOffset, longIterations);
assertTrue(equals(intValues, values));
} else {
intValues = null;
@@ -737,21 +743,21 @@ public class TestPackedInts extends Luce
// 3. re-encode
final long[] blocks2 = new long[blocksOffset2 + blocksLen];
- encoder.encode(values, valuesOffset, blocks2, blocksOffset2, iterations);
+ encoder.encode(values, valuesOffset, blocks2, blocksOffset2, longIterations);
assertArrayEquals(msg, Arrays.copyOfRange(blocks, blocksOffset, blocks.length),
Arrays.copyOfRange(blocks2, blocksOffset2, blocks2.length));
// test encoding from int[]
if (bpv <= 32) {
final long[] blocks3 = new long[blocks2.length];
- encoder.encode(intValues, valuesOffset, blocks3, blocksOffset2, iterations);
+ encoder.encode(intValues, valuesOffset, blocks3, blocksOffset2, longIterations);
assertArrayEquals(msg, blocks2, blocks3);
}
// 4. byte[] decoding
final byte[] byteBlocks = new byte[8 * blocks.length];
ByteBuffer.wrap(byteBlocks).asLongBuffer().put(blocks);
- final long[] values2 = new long[valuesOffset + iterations * valueCount];
- decoder.decode(byteBlocks, blocksOffset * 8, values2, valuesOffset, iterations);
+ final long[] values2 = new long[valuesOffset + longIterations * longValueCount];
+ decoder.decode(byteBlocks, blocksOffset * 8, values2, valuesOffset, byteIterations);
for (long value : values2) {
assertTrue(msg, value <= PackedInts.maxValue(bpv));
}
@@ -759,18 +765,18 @@ public class TestPackedInts extends Luce
// test decoding to int[]
if (bpv <= 32) {
final int[] intValues2 = new int[values2.length];
- decoder.decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, iterations);
+ decoder.decode(byteBlocks, blocksOffset * 8, intValues2, valuesOffset, byteIterations);
assertTrue(msg, equals(intValues2, values2));
}
// 5. byte[] encoding
final byte[] blocks3 = new byte[8 * (blocksOffset2 + blocksLen)];
- encoder.encode(values, valuesOffset, blocks3, 8 * blocksOffset2, iterations);
+ encoder.encode(values, valuesOffset, blocks3, 8 * blocksOffset2, byteIterations);
assertEquals(msg, LongBuffer.wrap(blocks2), ByteBuffer.wrap(blocks3).asLongBuffer());
// test encoding from int[]
if (bpv <= 32) {
final byte[] blocks4 = new byte[blocks3.length];
- encoder.encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, iterations);
+ encoder.encode(intValues, valuesOffset, blocks4, 8 * blocksOffset2, byteIterations);
assertArrayEquals(msg, blocks3, blocks4);
}
}