You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/02 21:58:23 UTC
svn commit: r1368678 - in
/lucene/dev/branches/pforcodec_3892/lucene/core/src:
java/org/apache/lucene/codecs/block/
java/org/apache/lucene/codecs/blockpacked/ test/org/apache/lucene/index/
Author: mikemccand
Date: Thu Aug 2 19:58:22 2012
New Revision: 1368678
URL: http://svn.apache.org/viewvc?rev=1368678&view=rev
Log:
LUCENE-4225: fix offset delta encoding to be startOffset - lastStartOffset, not startOffset - lastEndOffset
Modified:
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsReader.java Thu Aug 2 19:58:22 2012
@@ -958,7 +958,7 @@ public final class BlockPostingsReader e
private int payloadByteUpto;
private int payloadLength;
- private int lastEndOffset;
+ private int lastStartOffset;
private int startOffset;
private int endOffset;
@@ -1248,7 +1248,7 @@ public final class BlockPostingsReader e
}
position = 0;
payloadLength = 0;
- lastEndOffset = 0;
+ lastStartOffset = 0;
return doc;
}
@@ -1305,7 +1305,7 @@ public final class BlockPostingsReader e
// Skipper moved
if (DEBUG) {
- System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastEndOffset=" + lastEndOffset);
+ System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastStartOffset=" + lastStartOffset);
}
assert newDocUpto % blockSize == (blockSize-1): "got " + newDocUpto;
@@ -1318,7 +1318,7 @@ public final class BlockPostingsReader e
posPendingFP = skipper.getPosPointer();
payPendingFP = skipper.getPayPointer();
posPendingCount = skipper.getPosBufferUpto();
- lastEndOffset = skipper.getEndOffset();
+ lastStartOffset = skipper.getStartOffset();
payloadByteUpto = skipper.getPayloadByteUpto();
}
}
@@ -1359,7 +1359,7 @@ public final class BlockPostingsReader e
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
- lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
+ lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@@ -1386,11 +1386,11 @@ public final class BlockPostingsReader e
if (indexHasOffsets) {
// Must load offset blocks merely to sum
- // up into lastEndOffset:
+ // up into lastStartOffset:
readBlock(payIn, encoded, encodedBuffer, offsetStartDeltaBuffer);
readBlock(payIn, encoded, encodedBuffer, offsetLengthBuffer);
for(int i=0;i<blockSize;i++) {
- lastEndOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
+ lastStartOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
}
}
toSkip -= blockSize;
@@ -1403,7 +1403,7 @@ public final class BlockPostingsReader e
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
- lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
+ lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@@ -1414,7 +1414,9 @@ public final class BlockPostingsReader e
position = 0;
payloadLength = 0;
- lastEndOffset = 0;
+ // nocommit why carefully sum up lastStartOffset above
+ // only to set it to 0 now?
+ lastStartOffset = 0;
}
@Override
@@ -1467,9 +1469,9 @@ public final class BlockPostingsReader e
}
if (indexHasOffsets) {
- startOffset = lastEndOffset + offsetStartDeltaBuffer[posBufferUpto];
+ startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];
endOffset = startOffset + offsetLengthBuffer[posBufferUpto];
- lastEndOffset = endOffset;
+ lastStartOffset = startOffset;
}
posBufferUpto++;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockPostingsWriter.java Thu Aug 2 19:58:22 2012
@@ -95,11 +95,11 @@ public final class BlockPostingsWriter e
private long lastBlockPosFP;
private long lastBlockPayFP;
private int lastBlockPosBufferUpto;
- private int lastBlockEndOffset;
+ private int lastBlockStartOffset;
private int lastBlockPayloadByteUpto;
private int lastDocID;
private int lastPosition;
- private int lastEndOffset;
+ private int lastStartOffset;
private int docCount;
final byte[] encoded;
@@ -230,7 +230,7 @@ public final class BlockPostingsWriter e
lastBlockPayFP = payOut.getFilePointer();
}
lastBlockPosBufferUpto = posBufferUpto;
- lastBlockEndOffset = lastEndOffset;
+ lastBlockStartOffset = lastStartOffset;
lastBlockPayloadByteUpto = payloadByteUpto;
saveNextPosBlock = false;
if (DEBUG) {
@@ -269,7 +269,7 @@ public final class BlockPostingsWriter e
if (DEBUG) {
System.out.println(" bufferSkip at writeBlock: lastDocID=" + lastBlockDocID + " docCount=" + (docCount-blockSize));
}
- skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
+ skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
lastBlockDocID = docID;
saveNextPosBlock = true;
@@ -288,7 +288,7 @@ public final class BlockPostingsWriter e
}
lastPosition = 0;
- lastEndOffset = 0;
+ lastStartOffset = 0;
}
/** Add a new position & payload */
@@ -313,11 +313,11 @@ public final class BlockPostingsWriter e
}
if (fieldHasOffsets) {
- assert startOffset >= lastEndOffset;
+ assert startOffset >= lastStartOffset;
assert endOffset >= startOffset;
- offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastEndOffset;
+ offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
- lastEndOffset = endOffset;
+ lastStartOffset = startOffset;
}
posBufferUpto++;
@@ -388,7 +388,7 @@ public final class BlockPostingsWriter e
if (DEBUG) {
System.out.println(" bufferSkip at finishTerm: lastDocID=" + lastBlockDocID + " docCount=" + lastDocCount);
}
- skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
+ skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
if (DEBUG) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java Thu Aug 2 19:58:22 2012
@@ -37,12 +37,12 @@ final class BlockSkipReader extends Mult
private long posPointer[];
private long payPointer[];
private int posBufferUpto[];
- private int endOffset[];
+ private int startOffset[];
private int payloadByteUpto[];
private long lastPosPointer;
private long lastPayPointer;
- private int lastEndOffset;
+ private int lastStartOffset;
private int lastPayloadByteUpto;
private long lastDocPointer;
private int lastPosBufferUpto;
@@ -59,9 +59,9 @@ final class BlockSkipReader extends Mult
payloadByteUpto = null;
}
if (hasOffsets) {
- endOffset = new int[maxSkipLevels];
+ startOffset = new int[maxSkipLevels];
} else {
- endOffset = null;
+ startOffset = null;
}
if (hasOffsets || hasPayloads) {
payPointer = new long[maxSkipLevels];
@@ -108,8 +108,8 @@ final class BlockSkipReader extends Mult
return lastPayPointer;
}
- public int getEndOffset() {
- return lastEndOffset;
+ public int getStartOffset() {
+ return lastStartOffset;
}
public int getPayloadByteUpto() {
@@ -126,8 +126,8 @@ final class BlockSkipReader extends Mult
if (posPointer != null) {
posPointer[level] = lastPosPointer;
posBufferUpto[level] = lastPosBufferUpto;
- if (endOffset != null) {
- endOffset[level] = lastEndOffset;
+ if (startOffset != null) {
+ startOffset[level] = lastStartOffset;
}
if (payloadByteUpto != null) {
payloadByteUpto[level] = lastPayloadByteUpto;
@@ -155,8 +155,8 @@ final class BlockSkipReader extends Mult
if (payPointer != null) {
lastPayPointer = payPointer[level];
}
- if (endOffset != null) {
- lastEndOffset = endOffset[level];
+ if (startOffset != null) {
+ lastStartOffset = startOffset[level];
}
if (payloadByteUpto != null) {
lastPayloadByteUpto = payloadByteUpto[level];
@@ -192,8 +192,8 @@ final class BlockSkipReader extends Mult
payloadByteUpto[level] = skipStream.readVInt();
}
- if (endOffset != null) {
- endOffset[level] += skipStream.readVInt();
+ if (startOffset != null) {
+ startOffset[level] += skipStream.readVInt();
}
if (payPointer != null) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java Thu Aug 2 19:58:22 2012
@@ -34,7 +34,7 @@ final class BlockSkipWriter extends Mult
private long[] lastSkipDocPointer;
private long[] lastSkipPosPointer;
private long[] lastSkipPayPointer;
- private int[] lastEndOffset;
+ private int[] lastStartOffset;
private int[] lastPayloadByteUpto;
private final IndexOutput docOut;
@@ -46,7 +46,7 @@ final class BlockSkipWriter extends Mult
private long curPosPointer;
private long curPayPointer;
private int curPosBufferUpto;
- private int curEndOffset;
+ private int curStartOffset;
private int curPayloadByteUpto;
private boolean fieldHasPositions;
private boolean fieldHasOffsets;
@@ -65,7 +65,7 @@ final class BlockSkipWriter extends Mult
if (payOut != null) {
lastSkipPayPointer = new long[maxSkipLevels];
}
- lastEndOffset = new int[maxSkipLevels];
+ lastStartOffset = new int[maxSkipLevels];
lastPayloadByteUpto = new int[maxSkipLevels];
}
}
@@ -84,7 +84,7 @@ final class BlockSkipWriter extends Mult
if (fieldHasPositions) {
Arrays.fill(lastSkipPosPointer, posOut.getFilePointer());
if (fieldHasOffsets) {
- Arrays.fill(lastEndOffset, 0);
+ Arrays.fill(lastStartOffset, 0);
}
if (fieldHasPayloads) {
Arrays.fill(lastPayloadByteUpto, 0);
@@ -98,14 +98,14 @@ final class BlockSkipWriter extends Mult
/**
* Sets the values for the current skip data.
*/
- public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int endOffset, int payloadByteUpto) throws IOException {
+ public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int startOffset, int payloadByteUpto) throws IOException {
this.curDoc = doc;
this.curDocPointer = docOut.getFilePointer();
this.curPosPointer = posFP;
this.curPayPointer = payFP;
this.curPosBufferUpto = posBufferUpto;
this.curPayloadByteUpto = payloadByteUpto;
- this.curEndOffset = endOffset;
+ this.curStartOffset = startOffset;
bufferSkip(numDocs);
}
@@ -134,8 +134,8 @@ final class BlockSkipWriter extends Mult
}
if (fieldHasOffsets) {
- skipBuffer.writeVInt(curEndOffset - lastEndOffset[level]);
- lastEndOffset[level] = curEndOffset;
+ skipBuffer.writeVInt(curStartOffset - lastStartOffset[level]);
+ lastStartOffset[level] = curStartOffset;
}
if (fieldHasOffsets || fieldHasPayloads) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java Thu Aug 2 19:58:22 2012
@@ -978,7 +978,7 @@ public final class BlockPackedPostingsRe
private int payloadByteUpto;
private int payloadLength;
- private int lastEndOffset;
+ private int lastStartOffset;
private int startOffset;
private int endOffset;
@@ -1274,7 +1274,7 @@ public final class BlockPackedPostingsRe
}
position = 0;
payloadLength = 0;
- lastEndOffset = 0;
+ lastStartOffset = 0;
return doc;
}
@@ -1331,7 +1331,7 @@ public final class BlockPackedPostingsRe
// Skipper moved
if (DEBUG) {
- System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastEndOffset=" + lastEndOffset);
+ System.out.println(" skipper moved to docUpto=" + newDocUpto + " vs current=" + docUpto + "; docID=" + skipper.getDoc() + " fp=" + skipper.getDocPointer() + " pos.fp=" + skipper.getPosPointer() + " pos.bufferUpto=" + skipper.getPosBufferUpto() + " pay.fp=" + skipper.getPayPointer() + " lastStartOffset=" + lastStartOffset);
}
assert newDocUpto % blockSize == (blockSize-1): "got " + newDocUpto;
@@ -1344,7 +1344,7 @@ public final class BlockPackedPostingsRe
posPendingFP = skipper.getPosPointer();
payPendingFP = skipper.getPayPointer();
posPendingCount = skipper.getPosBufferUpto();
- lastEndOffset = skipper.getEndOffset();
+ lastStartOffset = skipper.getStartOffset();
payloadByteUpto = skipper.getPayloadByteUpto();
}
}
@@ -1385,7 +1385,7 @@ public final class BlockPackedPostingsRe
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
- lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
+ lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@@ -1412,11 +1412,11 @@ public final class BlockPackedPostingsRe
if (indexHasOffsets) {
// Must load offset blocks merely to sum
- // up into lastEndOffset:
+ // up into lastStartOffset:
readBlock(payIn, encoded, encodedBuffer, offsetStartDeltaLBuffer);
readBlock(payIn, encoded, encodedBuffer, offsetLengthLBuffer);
for(int i=0;i<blockSize;i++) {
- lastEndOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
+ lastStartOffset += offsetStartDeltaBuffer[i] + offsetLengthBuffer[i];
}
}
toSkip -= blockSize;
@@ -1429,7 +1429,7 @@ public final class BlockPackedPostingsRe
payloadByteUpto += payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
- lastEndOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
+ lastStartOffset += offsetStartDeltaBuffer[posBufferUpto] + offsetLengthBuffer[posBufferUpto];
}
posBufferUpto++;
}
@@ -1440,7 +1440,7 @@ public final class BlockPackedPostingsRe
position = 0;
payloadLength = 0;
- lastEndOffset = 0;
+ lastStartOffset = 0;
}
@Override
@@ -1493,9 +1493,9 @@ public final class BlockPackedPostingsRe
}
if (indexHasOffsets) {
- startOffset = lastEndOffset + (int)offsetStartDeltaBuffer[posBufferUpto];
+ startOffset = lastStartOffset + (int)offsetStartDeltaBuffer[posBufferUpto];
endOffset = startOffset + (int)offsetLengthBuffer[posBufferUpto];
- lastEndOffset = endOffset;
+ lastStartOffset = startOffset;
}
posBufferUpto++;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java Thu Aug 2 19:58:22 2012
@@ -102,11 +102,11 @@ public final class BlockPackedPostingsWr
private long lastBlockPosFP;
private long lastBlockPayFP;
private int lastBlockPosBufferUpto;
- private int lastBlockEndOffset;
+ private int lastBlockStartOffset;
private int lastBlockPayloadByteUpto;
private int lastDocID;
private int lastPosition;
- private int lastEndOffset;
+ private int lastStartOffset;
private int docCount;
final byte[] encoded;
@@ -249,7 +249,7 @@ public final class BlockPackedPostingsWr
lastBlockPayFP = payOut.getFilePointer();
}
lastBlockPosBufferUpto = posBufferUpto;
- lastBlockEndOffset = lastEndOffset;
+ lastBlockStartOffset = lastStartOffset;
lastBlockPayloadByteUpto = payloadByteUpto;
saveNextPosBlock = false;
if (DEBUG) {
@@ -288,7 +288,7 @@ public final class BlockPackedPostingsWr
if (DEBUG) {
System.out.println(" bufferSkip at writeBlock: lastDocID=" + lastBlockDocID + " docCount=" + (docCount-blockSize));
}
- skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
+ skipWriter.bufferSkip(lastBlockDocID, docCount-blockSize, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
lastBlockDocID = docID;
saveNextPosBlock = true;
@@ -307,7 +307,7 @@ public final class BlockPackedPostingsWr
}
lastPosition = 0;
- lastEndOffset = 0;
+ lastStartOffset = 0;
}
/** Add a new position & payload */
@@ -332,11 +332,11 @@ public final class BlockPackedPostingsWr
}
if (fieldHasOffsets) {
- assert startOffset >= lastEndOffset;
+ assert startOffset >= lastStartOffset;
assert endOffset >= startOffset;
- offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastEndOffset;
+ offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
offsetLengthBuffer[posBufferUpto] = endOffset - startOffset;
- lastEndOffset = endOffset;
+ lastStartOffset = startOffset;
}
posBufferUpto++;
@@ -407,7 +407,7 @@ public final class BlockPackedPostingsWr
if (DEBUG) {
System.out.println(" bufferSkip at finishTerm: lastDocID=" + lastBlockDocID + " docCount=" + lastDocCount);
}
- skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockEndOffset, lastBlockPayloadByteUpto);
+ skipWriter.bufferSkip(lastBlockDocID, lastDocCount, lastBlockPosFP, lastBlockPayFP, lastBlockPosBufferUpto, lastBlockStartOffset, lastBlockPayloadByteUpto);
}
if (DEBUG) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java Thu Aug 2 19:58:22 2012
@@ -37,12 +37,12 @@ final class BlockPackedSkipReader extend
private long posPointer[];
private long payPointer[];
private int posBufferUpto[];
- private int endOffset[];
+ private int startOffset[];
private int payloadByteUpto[];
private long lastPosPointer;
private long lastPayPointer;
- private int lastEndOffset;
+ private int lastStartOffset;
private int lastPayloadByteUpto;
private long lastDocPointer;
private int lastPosBufferUpto;
@@ -59,9 +59,9 @@ final class BlockPackedSkipReader extend
payloadByteUpto = null;
}
if (hasOffsets) {
- endOffset = new int[maxSkipLevels];
+ startOffset = new int[maxSkipLevels];
} else {
- endOffset = null;
+ startOffset = null;
}
if (hasOffsets || hasPayloads) {
payPointer = new long[maxSkipLevels];
@@ -108,8 +108,8 @@ final class BlockPackedSkipReader extend
return lastPayPointer;
}
- public int getEndOffset() {
- return lastEndOffset;
+ public int getStartOffset() {
+ return lastStartOffset;
}
public int getPayloadByteUpto() {
@@ -126,8 +126,8 @@ final class BlockPackedSkipReader extend
if (posPointer != null) {
posPointer[level] = lastPosPointer;
posBufferUpto[level] = lastPosBufferUpto;
- if (endOffset != null) {
- endOffset[level] = lastEndOffset;
+ if (startOffset != null) {
+ startOffset[level] = lastStartOffset;
}
if (payloadByteUpto != null) {
payloadByteUpto[level] = lastPayloadByteUpto;
@@ -155,8 +155,8 @@ final class BlockPackedSkipReader extend
if (payPointer != null) {
lastPayPointer = payPointer[level];
}
- if (endOffset != null) {
- lastEndOffset = endOffset[level];
+ if (startOffset != null) {
+ lastStartOffset = startOffset[level];
}
if (payloadByteUpto != null) {
lastPayloadByteUpto = payloadByteUpto[level];
@@ -192,8 +192,8 @@ final class BlockPackedSkipReader extend
payloadByteUpto[level] = skipStream.readVInt();
}
- if (endOffset != null) {
- endOffset[level] += skipStream.readVInt();
+ if (startOffset != null) {
+ startOffset[level] += skipStream.readVInt();
}
if (payPointer != null) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipWriter.java Thu Aug 2 19:58:22 2012
@@ -34,7 +34,7 @@ final class BlockPackedSkipWriter extend
private long[] lastSkipDocPointer;
private long[] lastSkipPosPointer;
private long[] lastSkipPayPointer;
- private int[] lastEndOffset;
+ private int[] lastStartOffset;
private int[] lastPayloadByteUpto;
private final IndexOutput docOut;
@@ -46,7 +46,7 @@ final class BlockPackedSkipWriter extend
private long curPosPointer;
private long curPayPointer;
private int curPosBufferUpto;
- private int curEndOffset;
+ private int curStartOffset;
private int curPayloadByteUpto;
private boolean fieldHasPositions;
private boolean fieldHasOffsets;
@@ -65,7 +65,7 @@ final class BlockPackedSkipWriter extend
if (payOut != null) {
lastSkipPayPointer = new long[maxSkipLevels];
}
- lastEndOffset = new int[maxSkipLevels];
+ lastStartOffset = new int[maxSkipLevels];
lastPayloadByteUpto = new int[maxSkipLevels];
}
}
@@ -84,7 +84,7 @@ final class BlockPackedSkipWriter extend
if (fieldHasPositions) {
Arrays.fill(lastSkipPosPointer, posOut.getFilePointer());
if (fieldHasOffsets) {
- Arrays.fill(lastEndOffset, 0);
+ Arrays.fill(lastStartOffset, 0);
}
if (fieldHasPayloads) {
Arrays.fill(lastPayloadByteUpto, 0);
@@ -98,14 +98,14 @@ final class BlockPackedSkipWriter extend
/**
* Sets the values for the current skip data.
*/
- public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int endOffset, int payloadByteUpto) throws IOException {
+ public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int startOffset, int payloadByteUpto) throws IOException {
this.curDoc = doc;
this.curDocPointer = docOut.getFilePointer();
this.curPosPointer = posFP;
this.curPayPointer = payFP;
this.curPosBufferUpto = posBufferUpto;
this.curPayloadByteUpto = payloadByteUpto;
- this.curEndOffset = endOffset;
+ this.curStartOffset = startOffset;
bufferSkip(numDocs);
}
@@ -134,8 +134,8 @@ final class BlockPackedSkipWriter extend
}
if (fieldHasOffsets) {
- skipBuffer.writeVInt(curEndOffset - lastEndOffset[level]);
- lastEndOffset[level] = curEndOffset;
+ skipBuffer.writeVInt(curStartOffset - lastStartOffset[level]);
+ lastStartOffset[level] = curStartOffset;
}
if (fieldHasOffsets || fieldHasPayloads) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java?rev=1368678&r1=1368677&r2=1368678&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java Thu Aug 2 19:58:22 2012
@@ -439,6 +439,14 @@ public class TestPostingsOffsets extends
}
}
+ public void testStackedTokens() throws Exception {
+ checkTokens(new Token[] {
+ makeToken("foo", 1, 0, 3),
+ makeToken("foo", 0, 0, 3),
+ makeToken("foo", 0, 0, 3)
+ });
+ }
+
public void testLegalbutVeryLargeOffsets() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));