You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/01/30 14:21:17 UTC
svn commit: r1440397 - in /lucene/dev/branches/lucene4547/lucene:
core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
core/src/java/org/apache/lucene/util/ByteBlockPool.java
suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java
Author: mikemccand
Date: Wed Jan 30 13:21:17 2013
New Revision: 1440397
URL: http://svn.apache.org/viewvc?rev=1440397&view=rev
Log:
fix BinaryDVWriter to use ByteBlockPool
Modified:
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java
lucene/dev/branches/lucene4547/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java?rev=1440397&r1=1440396&r2=1440397&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java Wed Jan 30 13:21:17 2013
@@ -17,38 +17,32 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
-
import java.io.IOException;
import java.util.Iterator;
import java.util.NoSuchElementException;
import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.store.RAMFile;
-import org.apache.lucene.store.RAMInputStream;
-import org.apache.lucene.store.RAMOutputStream;
+import org.apache.lucene.util.ByteBlockPool.DirectTrackingAllocator;
+import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.packed.AppendingLongBuffer;
+import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
+
/** Buffers up pending byte[] per doc, then flushes when
* segment flushes. */
class BinaryDocValuesWriter extends DocValuesWriter {
- private final RAMFile bytes;
- private final RAMOutputStream bytesWriter;
+ private final ByteBlockPool pool;
private final AppendingLongBuffer lengths;
private final FieldInfo fieldInfo;
- private final Counter iwBytesUsed;
- private long bytesUsed;
private int addedValues = 0;
public BinaryDocValuesWriter(FieldInfo fieldInfo, Counter iwBytesUsed) {
this.fieldInfo = fieldInfo;
- this.bytes = new RAMFile();
- this.bytesWriter = new RAMOutputStream(bytes);
- this.iwBytesUsed = iwBytesUsed;
+ this.pool = new ByteBlockPool(new DirectTrackingAllocator(iwBytesUsed));
this.lengths = new AppendingLongBuffer();
}
@@ -70,31 +64,11 @@ class BinaryDocValuesWriter extends DocV
}
addedValues++;
lengths.add(value.length);
- try {
- bytesWriter.writeBytes(value.bytes, value.offset, value.length);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
-
- updateBytesUsed();
- }
-
- private void updateBytesUsed() {
- // nocommit not totally accurate, but just fix not to use RAMFile anyway
- long numBuffers = (bytesWriter.getFilePointer() / 1024) + 1; // round up
- long oversize = numBuffers * (1024 + 32); // fudge for arraylist/etc overhead
- final long newBytesUsed = lengths.ramBytesUsed() + oversize;
- iwBytesUsed.addAndGet(newBytesUsed - bytesUsed);
- bytesUsed = newBytesUsed;
+ pool.append(value);
}
@Override
public void finish(int maxDoc) {
- try {
- bytesWriter.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
}
@Override
@@ -119,16 +93,11 @@ class BinaryDocValuesWriter extends DocV
final AppendingLongBuffer.Iterator lengthsIterator = lengths.iterator();
final int size = lengths.size();
final int maxDoc;
- final RAMInputStream bytesReader;
int upto;
+ long byteOffset;
BytesIterator(int maxDoc) {
this.maxDoc = maxDoc;
- try {
- bytesReader = new RAMInputStream("BinaryDocValuesWriter", bytes);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
}
@Override
@@ -143,14 +112,11 @@ class BinaryDocValuesWriter extends DocV
}
if (upto < size) {
int length = (int) lengthsIterator.next();
- value.grow(length);
- try {
- bytesReader.readBytes(value.bytes, 0, length);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- value.length = length;
+ pool.readBytes(value, byteOffset, length);
+ byteOffset += length;
} else {
+ // This is to handle last N documents not having
+ // this DV field in the end of the segment:
value.length = 0;
}
upto++;
Modified: lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java?rev=1440397&r1=1440396&r2=1440397&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java (original)
+++ lucene/dev/branches/lucene4547/lucene/core/src/java/org/apache/lucene/util/ByteBlockPool.java Wed Jan 30 13:21:17 2013
@@ -253,8 +253,9 @@ public final class ByteBlockPool {
final int newSize = LEVEL_SIZE_ARRAY[newLevel];
// Maybe allocate another block
- if (byteUpto > BYTE_BLOCK_SIZE-newSize)
+ if (byteUpto > BYTE_BLOCK_SIZE-newSize) {
nextBuffer();
+ }
final int newUpto = byteUpto;
final int offset = newUpto + byteOffset;
@@ -280,7 +281,7 @@ public final class ByteBlockPool {
// Fill in a BytesRef from term's length & bytes encoded in
// byte block
- public final void setBytesRef(BytesRef term, int textStart) {
+ public void setBytesRef(BytesRef term, int textStart) {
final byte[] bytes = term.bytes = buffers[textStart >> BYTE_BLOCK_SHIFT];
int pos = textStart & BYTE_BLOCK_MASK;
if ((bytes[pos] & 0x80) == 0) {
@@ -296,11 +297,14 @@ public final class ByteBlockPool {
}
/**
- * Copies the given {@link BytesRef} at the current positions (
- * {@link #byteUpto} across buffer boundaries
+ * Appends the bytes in the provided {@link BytesRef} at
+ * the current position.
*/
- public final void copy(final BytesRef bytes) {
+ public void append(final BytesRef bytes) {
int length = bytes.length;
+ if (length == 0) {
+ return;
+ }
int offset = bytes.offset;
int overflow = (length + byteUpto) - BYTE_BLOCK_SIZE;
do {
@@ -310,9 +314,11 @@ public final class ByteBlockPool {
break;
} else {
final int bytesToCopy = length-overflow;
- System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy);
- offset += bytesToCopy;
- length -= bytesToCopy;
+ if (bytesToCopy > 0) {
+ System.arraycopy(bytes.bytes, offset, buffer, byteUpto, bytesToCopy);
+ offset += bytesToCopy;
+ length -= bytesToCopy;
+ }
nextBuffer();
overflow = overflow - BYTE_BLOCK_SIZE;
}
@@ -320,17 +326,20 @@ public final class ByteBlockPool {
}
/**
- * Copies bytes from the pool starting at the given offset with the given
+ * Reads bytes bytes out of the pool starting at the given offset with the given
* length into the given {@link BytesRef} at offset <tt>0</tt>.
* <p>Note: this method allows to copy across block boundaries.</p>
*/
- public final void copyFrom(final BytesRef bytes, final int offset, final int length) {
+ public void readBytes(final BytesRef bytes, final long offset, final int length) {
bytes.offset = 0;
bytes.grow(length);
bytes.length = length;
- int bufferIndex = offset >> BYTE_BLOCK_SHIFT;
+ if (length == 0) {
+ return;
+ }
+ int bufferIndex = (int) (offset >> BYTE_BLOCK_SHIFT);
byte[] buffer = buffers[bufferIndex];
- int pos = offset & BYTE_BLOCK_MASK;
+ int pos = (int) (offset & BYTE_BLOCK_MASK);
int overflow = (pos + length) - BYTE_BLOCK_SIZE;
do {
if (overflow <= 0) {
Modified: lucene/dev/branches/lucene4547/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4547/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java?rev=1440397&r1=1440396&r2=1440397&view=diff
==============================================================================
--- lucene/dev/branches/lucene4547/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java (original)
+++ lucene/dev/branches/lucene4547/lucene/suggest/src/java/org/apache/lucene/search/suggest/BytesRefArray.java Wed Jan 30 13:21:17 2013
@@ -79,7 +79,7 @@ final class BytesRefArray {
bytesUsed.addAndGet((offsets.length - oldLen)
* RamUsageEstimator.NUM_BYTES_INT);
}
- pool.copy(bytes);
+ pool.append(bytes);
offsets[lastElement++] = currentOffset;
currentOffset += bytes.length;
return lastElement;
@@ -104,7 +104,7 @@ final class BytesRefArray {
int offset = offsets[ord];
int length = ord == lastElement - 1 ? currentOffset - offset
: offsets[ord + 1] - offset;
- pool.copyFrom(spare, offset, length);
+ pool.readBytes(spare, offset, length);
return spare;
}
throw new IndexOutOfBoundsException("index " + ord