You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2014/06/16 16:22:03 UTC
svn commit: r1602873 - in /lucene/dev/trunk/lucene:
codecs/src/java/org/apache/lucene/codecs/memory/
core/src/java/org/apache/lucene/codecs/lucene45/
core/src/java/org/apache/lucene/codecs/lucene49/
core/src/java/org/apache/lucene/util/packed/ core/src...
Author: jpountz
Date: Mon Jun 16 14:22:02 2014
New Revision: 1602873
URL: http://svn.apache.org/r1602873
Log:
LUCENE-5759: Add PackedInts.unsignedBitsRequired.
Modified:
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryDocValuesConsumer.java Mon Jun 16 14:22:02 2014
@@ -146,7 +146,7 @@ class MemoryDocValuesConsumer extends Do
++count;
if (count % BLOCK_SIZE == 0) {
final long blockDelta = currentBlockMax - currentBlockMin;
- final int blockDeltaRequired = blockDelta < 0 ? 64 : PackedInts.bitsRequired(blockDelta);
+ final int blockDeltaRequired = PackedInts.unsignedBitsRequired(blockDelta);
final int blockBPV = PackedInts.fastestFormatAndBits(BLOCK_SIZE, blockDeltaRequired, acceptableOverheadRatio).bitsPerValue;
blockSum += blockBPV;
currentBlockMax = Long.MIN_VALUE;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java Mon Jun 16 14:22:02 2014
@@ -147,7 +147,7 @@ class Lucene45DocValuesConsumer extends
final int format;
if (uniqueValues != null
- && (delta < 0L || PackedInts.bitsRequired(uniqueValues.size() - 1) < PackedInts.bitsRequired(delta))
+ && (PackedInts.bitsRequired(uniqueValues.size() - 1) < PackedInts.unsignedBitsRequired(delta))
&& count <= Integer.MAX_VALUE) {
format = TABLE_COMPRESSED;
} else if (gcd != 0 && gcd != 1) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene49/Lucene49DocValuesConsumer.java Mon Jun 16 14:22:02 2014
@@ -148,14 +148,17 @@ class Lucene49DocValuesConsumer extends
}
final long delta = maxValue - minValue;
- final int deltaBitsRequired = delta < 0 ? 64 : DirectWriter.bitsRequired(delta);
+ final int deltaBitsRequired = DirectWriter.unsignedBitsRequired(delta);
+ final int tableBitsRequired = uniqueValues == null
+ ? Integer.MAX_VALUE
+ : DirectWriter.bitsRequired(uniqueValues.size() - 1);
final int format;
- if (uniqueValues != null && DirectWriter.bitsRequired(uniqueValues.size() - 1) < deltaBitsRequired) {
+ if (uniqueValues != null && tableBitsRequired < deltaBitsRequired) {
format = TABLE_COMPRESSED;
} else if (gcd != 0 && gcd != 1) {
final long gcdDelta = (maxValue - minValue) / gcd;
- final long gcdBitsRequired = gcdDelta < 0 ? 64 : DirectWriter.bitsRequired(gcdDelta);
+ final long gcdBitsRequired = DirectWriter.unsignedBitsRequired(gcdDelta);
format = gcdBitsRequired < deltaBitsRequired ? GCD_COMPRESSED : DELTA_COMPRESSED;
} else {
format = DELTA_COMPRESSED;
@@ -177,7 +180,7 @@ class Lucene49DocValuesConsumer extends
meta.writeLong(minValue);
meta.writeLong(gcd);
final long maxDelta = (maxValue - minValue) / gcd;
- final int bits = maxDelta < 0 ? 64 : DirectWriter.bitsRequired(maxDelta);
+ final int bits = DirectWriter.unsignedBitsRequired(maxDelta);
meta.writeVInt(bits);
final DirectWriter quotientWriter = DirectWriter.getInstance(data, count, bits);
for (Number nv : values) {
@@ -206,9 +209,8 @@ class Lucene49DocValuesConsumer extends
meta.writeLong(decode[i]);
encode.put(decode[i], i);
}
- final int bitsRequired = DirectWriter.bitsRequired(uniqueValues.size() - 1);
- meta.writeVInt(bitsRequired);
- final DirectWriter ordsWriter = DirectWriter.getInstance(data, count, bitsRequired);
+ meta.writeVInt(tableBitsRequired);
+ final DirectWriter ordsWriter = DirectWriter.getInstance(data, count, tableBitsRequired);
for (Number nv : values) {
ordsWriter.add(encode.get(nv == null ? 0 : nv.longValue()));
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingDeltaPackedLongBuffer.java Mon Jun 16 14:22:02 2014
@@ -104,7 +104,7 @@ public final class AppendingDeltaPackedL
values[valuesOff] = new PackedInts.NullReader(pendingOff);
} else {
// build a new packed reader
- final int bitsRequired = delta < 0 ? 64 : PackedInts.bitsRequired(delta);
+ final int bitsRequired = PackedInts.unsignedBitsRequired(delta);
for (int i = 0; i < pendingOff; ++i) {
pending[i] -= minValue;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedWriter.java Mon Jun 16 14:22:02 2014
@@ -76,7 +76,7 @@ public final class BlockPackedWriter ext
}
final long delta = max - min;
- int bitsRequired = delta < 0 ? 64 : delta == 0L ? 0 : PackedInts.bitsRequired(delta);
+ int bitsRequired = delta == 0 ? 0 : PackedInts.unsignedBitsRequired(delta);
if (bitsRequired == 64) {
// no need to delta-encode
min = 0L;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperationPacked.java Mon Jun 16 14:22:02 2014
@@ -221,7 +221,7 @@ class BulkOperationPacked extends BulkOp
int bitsLeft = 8;
for (int i = 0; i < byteValueCount * iterations; ++i) {
final long v = values[valuesOffset++];
- assert bitsPerValue == 64 || PackedInts.bitsRequired(v) <= bitsPerValue;
+ assert PackedInts.unsignedBitsRequired(v) <= bitsPerValue;
if (bitsPerValue < bitsLeft) {
// just buffer
nextBlock |= v << (bitsLeft - bitsPerValue);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/DirectWriter.java Mon Jun 16 14:22:02 2014
@@ -111,17 +111,14 @@ public final class DirectWriter {
}
/**
- * Returns how many bits are required to hold values up
- * to and including maxValue
+ * Round a number of bits per value to the next amount of bits per value that
+ * is supported by this writer.
*
- * @param maxValue the maximum value that should be representable.
- * @return the amount of bits needed to represent values from 0 to maxValue.
+ * @param bitsRequired the amount of bits required
+ * @return the next number of bits per value that is gte the provided value
+ * and supported by this writer
*/
- public static int bitsRequired(long maxValue) {
- if (maxValue < 0) {
- throw new IllegalArgumentException("maxValue must be non-negative (got: " + maxValue + ")");
- }
- int bitsRequired = Math.max(1, 64 - Long.numberOfLeadingZeros(maxValue));
+ private static int roundBits(int bitsRequired) {
int index = Arrays.binarySearch(SUPPORTED_BITS_PER_VALUE, bitsRequired);
if (index < 0) {
return SUPPORTED_BITS_PER_VALUE[-index-1];
@@ -129,7 +126,31 @@ public final class DirectWriter {
return bitsRequired;
}
}
-
+
+ /**
+ * Returns how many bits are required to hold values up
+ * to and including maxValue
+ *
+ * @param maxValue the maximum value that should be representable.
+ * @return the amount of bits needed to represent values from 0 to maxValue.
+ * @see PackedInts#bitsRequired(long)
+ */
+ public static int bitsRequired(long maxValue) {
+ return roundBits(PackedInts.bitsRequired(maxValue));
+ }
+
+ /**
+ * Returns how many bits are required to hold values up
+ * to and including maxValue, interpreted as an unsigned value.
+ *
+ * @param maxValue the maximum value that should be representable.
+ * @return the amount of bits needed to represent values from 0 to maxValue.
+ * @see PackedInts#unsignedBitsRequired(long)
+ */
+ public static int unsignedBitsRequired(long maxValue) {
+ return roundBits(PackedInts.unsignedBitsRequired(maxValue));
+ }
+
final static int SUPPORTED_BITS_PER_VALUE[] = new int[] {
1, 2, 4, 8, 12, 16, 20, 24, 28, 32, 40, 48, 56, 64
};
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/GrowableWriter.java Mon Jun 16 14:22:02 2014
@@ -74,7 +74,7 @@ public class GrowableWriter extends Pack
if ((value & currentMask) == value) {
return;
}
- final int bitsRequired = value < 0 ? 64 : PackedInts.bitsRequired(value);
+ final int bitsRequired = PackedInts.unsignedBitsRequired(value);
assert bitsRequired > current.getBitsPerValue();
final int valueCount = size();
PackedInts.Mutable next = PackedInts.getMutable(valueCount, bitsRequired, acceptableOverheadRatio);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java Mon Jun 16 14:22:02 2014
@@ -125,7 +125,7 @@ public final class MonotonicAppendingLon
if (maxDelta == 0) {
values[valuesOff] = new PackedInts.NullReader(pendingOff);
} else {
- final int bitsRequired = maxDelta < 0 ? 64 : PackedInts.bitsRequired(maxDelta);
+ final int bitsRequired = PackedInts.unsignedBitsRequired(maxDelta);
final PackedInts.Mutable mutable = PackedInts.getMutable(pendingOff, bitsRequired, acceptableOverheadRatio);
for (int i = 0; i < pendingOff; ) {
i += mutable.set(i, pending, i, pendingOff - i);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java Mon Jun 16 14:22:02 2014
@@ -254,7 +254,7 @@ class Packed64 extends PackedInts.Mutabl
@Override
public void fill(int fromIndex, int toIndex, long val) {
- assert PackedInts.bitsRequired(val) <= getBitsPerValue();
+ assert PackedInts.unsignedBitsRequired(val) <= getBitsPerValue();
assert fromIndex <= toIndex;
// minimum number of values that use an exact number of full blocks
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java Mon Jun 16 14:22:02 2014
@@ -159,7 +159,7 @@ abstract class Packed64SingleBlock exten
public void fill(int fromIndex, int toIndex, long val) {
assert fromIndex >= 0;
assert fromIndex <= toIndex;
- assert PackedInts.bitsRequired(val) <= bitsPerValue;
+ assert PackedInts.unsignedBitsRequired(val) <= bitsPerValue;
final int valuesPerBlock = 64 / bitsPerValue;
if (toIndex - fromIndex <= valuesPerBlock << 1) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Mon Jun 16 14:22:02 2014
@@ -1131,6 +1131,7 @@ public class PackedInts {
/** Returns how many bits are required to hold values up
* to and including maxValue
+ * NOTE: This method returns at least 1.
* @param maxValue the maximum value that should be representable.
* @return the amount of bits needed to represent values from 0 to maxValue.
* @lucene.internal
@@ -1139,7 +1140,16 @@ public class PackedInts {
if (maxValue < 0) {
throw new IllegalArgumentException("maxValue must be non-negative (got: " + maxValue + ")");
}
- return Math.max(1, 64 - Long.numberOfLeadingZeros(maxValue));
+ return unsignedBitsRequired(maxValue);
+ }
+
+ /** Returns how many bits are required to store <code>bits</code>,
+ * interpreted as an unsigned value.
+ * NOTE: This method returns at least 1.
+ * @lucene.internal
+ */
+ public static int unsignedBitsRequired(long bits) {
+ return Math.max(1, 64 - Long.numberOfLeadingZeros(bits));
}
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java Mon Jun 16 14:22:02 2014
@@ -56,7 +56,7 @@ final class PackedWriter extends PackedI
@Override
public void add(long v) throws IOException {
- assert bitsPerValue == 64 || (v >= 0 && v <= PackedInts.maxValue(bitsPerValue)) : bitsPerValue;
+ assert PackedInts.unsignedBitsRequired(v) <= bitsPerValue;
assert !finished;
if (valueCount != -1 && written >= valueCount) {
throw new EOFException("Writing past end of stream");
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestDirectPacked.java Mon Jun 16 14:22:02 2014
@@ -25,7 +25,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.MMapDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.packed.DirectReader;
import org.apache.lucene.util.packed.DirectWriter;
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1602873&r1=1602872&r2=1602873&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Mon Jun 16 14:22:02 2014
@@ -70,6 +70,9 @@ public class TestPackedInts extends Luce
assertEquals(61, PackedInts.bitsRequired(0x1FFFFFFFFFFFFFFFL));
assertEquals(62, PackedInts.bitsRequired(0x3FFFFFFFFFFFFFFFL));
assertEquals(63, PackedInts.bitsRequired(0x7FFFFFFFFFFFFFFFL));
+ assertEquals(64, PackedInts.unsignedBitsRequired(-1));
+ assertEquals(64, PackedInts.unsignedBitsRequired(Long.MIN_VALUE));
+ assertEquals(1, PackedInts.bitsRequired(0));
}
public void testMaxValues() {