You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/06/07 01:30:16 UTC
svn commit: r1490469 - in /lucene/dev/trunk/lucene/core/src:
java/org/apache/lucene/util/packed/ test/org/apache/lucene/util/packed/
Author: jpountz
Date: Thu Jun 6 23:30:15 2013
New Revision: 1490469
URL: http://svn.apache.org/r1490469
Log:
LUCENE-5034: Make (Monotonic)AppendingLongBuffer's block size configurable.
Modified:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java?rev=1490469&r1=1490468&r2=1490469&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AbstractAppendingLongBuffer.java Thu Jun 6 23:30:15 2013
@@ -17,6 +17,8 @@ package org.apache.lucene.util.packed;
* limitations under the License.
*/
+import static org.apache.lucene.util.packed.PackedInts.checkBlockSize;
+
import java.util.Arrays;
import org.apache.lucene.util.ArrayUtil;
@@ -25,33 +27,37 @@ import org.apache.lucene.util.RamUsageEs
/** Common functionality shared by {@link AppendingLongBuffer} and {@link MonotonicAppendingLongBuffer}. */
abstract class AbstractAppendingLongBuffer {
- static final int BLOCK_BITS = 10;
- static final int MAX_PENDING_COUNT = 1 << BLOCK_BITS;
- static final int BLOCK_MASK = MAX_PENDING_COUNT - 1;
+ static final int MIN_PAGE_SIZE = 64;
+ // More than 1M doesn't really makes sense with these appending buffers
+ // since their goal is to try to have small numbers of bits per value
+ static final int MAX_PAGE_SIZE = 1 << 20;
+ final int pageShift, pageMask;
long[] minValues;
PackedInts.Reader[] deltas;
private long deltasBytes;
int valuesOff;
- long[] pending;
+ final long[] pending;
int pendingOff;
- AbstractAppendingLongBuffer(int initialBlockCount) {
- minValues = new long[16];
- deltas = new PackedInts.Reader[16];
- pending = new long[MAX_PENDING_COUNT];
+ AbstractAppendingLongBuffer(int initialBlockCount, int pageSize) {
+ minValues = new long[initialBlockCount];
+ deltas = new PackedInts.Reader[initialBlockCount];
+ pending = new long[pageSize];
+ pageShift = checkBlockSize(pageSize, MIN_PAGE_SIZE, MAX_PAGE_SIZE);
+ pageMask = pageSize - 1;
valuesOff = 0;
pendingOff = 0;
}
/** Get the number of values that have been added to the buffer. */
public final long size() {
- return valuesOff * (long) MAX_PENDING_COUNT + pendingOff;
+ return valuesOff * (long) pending.length + pendingOff;
}
/** Append a value to this buffer. */
public final void add(long l) {
- if (pendingOff == MAX_PENDING_COUNT) {
+ if (pendingOff == pending.length) {
// check size
if (deltas.length == valuesOff) {
final int newLength = ArrayUtil.oversize(valuesOff + 1, 8);
@@ -80,8 +86,8 @@ abstract class AbstractAppendingLongBuff
if (index < 0 || index >= size()) {
throw new IndexOutOfBoundsException("" + index);
}
- int block = (int) (index >> BLOCK_BITS);
- int element = (int) (index & BLOCK_MASK);
+ final int block = (int) (index >> pageShift);
+ final int element = (int) (index & pageMask);
return get(block, element);
}
@@ -99,7 +105,7 @@ abstract class AbstractAppendingLongBuff
if (valuesOff == 0) {
currentValues = pending;
} else {
- currentValues = new long[MAX_PENDING_COUNT];
+ currentValues = new long[pending.length];
fillValues();
}
}
@@ -115,7 +121,7 @@ abstract class AbstractAppendingLongBuff
public final long next() {
assert hasNext();
long result = currentValues[pOff++];
- if (pOff == MAX_PENDING_COUNT) {
+ if (pOff == pending.length) {
vOff += 1;
pOff = 0;
if (vOff <= valuesOff) {
@@ -139,6 +145,7 @@ abstract class AbstractAppendingLongBuff
public long ramBytesUsed() {
// TODO: this is called per-doc-per-norms/dv-field, can we optimize this?
long bytesUsed = RamUsageEstimator.alignObjectSize(baseRamBytesUsed())
+ + 2 * RamUsageEstimator.NUM_BYTES_INT // pageShift, pageMask
+ RamUsageEstimator.NUM_BYTES_LONG // valuesBytes
+ RamUsageEstimator.sizeOf(pending)
+ RamUsageEstimator.sizeOf(minValues)
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java?rev=1490469&r1=1490468&r2=1490469&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/AppendingLongBuffer.java Thu Jun 6 23:30:15 2013
@@ -27,9 +27,16 @@ import java.util.Arrays;
*/
public final class AppendingLongBuffer extends AbstractAppendingLongBuffer {
- /** Sole constructor. */
+ /** @param initialPageCount the initial number of pages
+ * @param pageSize the size of a single page */
+ public AppendingLongBuffer(int initialPageCount, int pageSize) {
+ super(initialPageCount, pageSize);
+ }
+
+ /** Create an {@link AppendingLongBuffer} with initialPageCount=16 and
+ * pageSize=1024. */
public AppendingLongBuffer() {
- super(16);
+ this(16, 1024);
}
@Override
@@ -43,8 +50,9 @@ public final class AppendingLongBuffer e
}
}
+ @Override
void packPendingValues() {
- assert pendingOff == MAX_PENDING_COUNT;
+ assert pendingOff == pending.length;
// compute max delta
long minValue = pending[0];
@@ -71,6 +79,7 @@ public final class AppendingLongBuffer e
}
/** Return an iterator over the values of this buffer. */
+ @Override
public Iterator iterator() {
return new Iterator();
}
@@ -78,20 +87,21 @@ public final class AppendingLongBuffer e
/** A long iterator. */
public final class Iterator extends AbstractAppendingLongBuffer.Iterator {
- private Iterator() {
+ Iterator() {
super();
}
+ @Override
void fillValues() {
if (vOff == valuesOff) {
currentValues = pending;
} else if (deltas[vOff] == null) {
Arrays.fill(currentValues, minValues[vOff]);
} else {
- for (int k = 0; k < MAX_PENDING_COUNT; ) {
- k += deltas[vOff].get(k, currentValues, k, MAX_PENDING_COUNT - k);
+ for (int k = 0; k < pending.length; ) {
+ k += deltas[vOff].get(k, currentValues, k, pending.length - k);
}
- for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
+ for (int k = 0; k < pending.length; ++k) {
currentValues[k] += minValues[vOff];
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java?rev=1490469&r1=1490468&r2=1490469&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/packed/MonotonicAppendingLongBuffer.java Thu Jun 6 23:30:15 2013
@@ -37,14 +37,22 @@ public final class MonotonicAppendingLon
return (n >> 63) ^ (n << 1);
}
- private float[] averages;
+ float[] averages;
- /** Sole constructor. */
+ /** @param initialPageCount the initial number of pages
+ * @param pageSize the size of a single page */
+ public MonotonicAppendingLongBuffer(int initialPageCount, int pageSize) {
+ super(initialPageCount, pageSize);
+ averages = new float[pending.length];
+ }
+
+ /** Create an {@link MonotonicAppendingLongBuffer} with initialPageCount=16
+ * and pageSize=1024. */
public MonotonicAppendingLongBuffer() {
- super(16);
- averages = new float[16];
+ this(16, 1024);
}
-
+
+ @Override
long get(int block, int element) {
if (block == valuesOff) {
return pending[element];
@@ -66,16 +74,16 @@ public final class MonotonicAppendingLon
@Override
void packPendingValues() {
- assert pendingOff == MAX_PENDING_COUNT;
+ assert pendingOff == pending.length;
minValues[valuesOff] = pending[0];
- averages[valuesOff] = (float) (pending[BLOCK_MASK] - pending[0]) / BLOCK_MASK;
+ averages[valuesOff] = (float) (pending[pending.length - 1] - pending[0]) / (pending.length - 1);
- for (int i = 0; i < MAX_PENDING_COUNT; ++i) {
+ for (int i = 0; i < pending.length; ++i) {
pending[i] = zigZagEncode(pending[i] - minValues[valuesOff] - (long) (averages[valuesOff] * (long) i));
}
long maxDelta = 0;
- for (int i = 0; i < MAX_PENDING_COUNT; ++i) {
+ for (int i = 0; i < pending.length; ++i) {
if (pending[i] < 0) {
maxDelta = -1;
break;
@@ -94,6 +102,7 @@ public final class MonotonicAppendingLon
}
/** Return an iterator over the values of this buffer. */
+ @Override
public Iterator iterator() {
return new Iterator();
}
@@ -105,18 +114,19 @@ public final class MonotonicAppendingLon
super();
}
+ @Override
void fillValues() {
if (vOff == valuesOff) {
currentValues = pending;
} else if (deltas[vOff] == null) {
- for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
+ for (int k = 0; k < pending.length; ++k) {
currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k);
}
} else {
- for (int k = 0; k < MAX_PENDING_COUNT; ) {
- k += deltas[vOff].get(k, currentValues, k, MAX_PENDING_COUNT - k);
+ for (int k = 0; k < pending.length; ) {
+ k += deltas[vOff].get(k, currentValues, k, pending.length - k);
}
- for (int k = 0; k < MAX_PENDING_COUNT; ++k) {
+ for (int k = 0; k < pending.length; ++k) {
currentValues[k] = minValues[vOff] + (long) (averages[vOff] * (long) k) + zigZagDecode(currentValues[k]);
}
}
Modified: lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java?rev=1490469&r1=1490468&r2=1490469&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java (original)
+++ lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java Thu Jun 6 23:30:15 2013
@@ -863,13 +863,15 @@ public class TestPackedInts extends Luce
final long[] arr = new long[RandomInts.randomIntBetween(random(), 1, 1000000)];
for (int bpv : new int[] {0, 1, 63, 64, RandomInts.randomIntBetween(random(), 2, 62)}) {
for (boolean monotonic : new boolean[] {true, false}) {
+ final int pageSize = 1 << _TestUtil.nextInt(random(), 6, 20);
+ final int initialPageCount = _TestUtil.nextInt(random(), 0, 16);
AbstractAppendingLongBuffer buf;
final int inc;
if (monotonic) {
- buf = new MonotonicAppendingLongBuffer();
+ buf = new MonotonicAppendingLongBuffer(initialPageCount, pageSize);
inc = _TestUtil.nextInt(random(), -1000, 1000);
} else {
- buf = new AppendingLongBuffer();
+ buf = new AppendingLongBuffer(initialPageCount, pageSize);
inc = 0;
}
if (bpv == 0) {