You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2012/08/08 19:28:17 UTC
svn commit: r1370850 - in
/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked:
BlockPackedPostingsReader.java BlockPackedPostingsWriter.java
BlockPackedSkipReader.java ForUtil.java
Author: jpountz
Date: Wed Aug 8 17:28:16 2012
New Revision: 1370850
URL: http://svn.apache.org/viewvc?rev=1370850&view=rev
Log:
LUCENE-3892: Backport of Mike's last changes + removal of unnecessary casts.
Modified:
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java?rev=1370850&r1=1370849&r2=1370850&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsReader.java Wed Aug 8 17:28:16 2012
@@ -328,6 +328,10 @@ public final class BlockPackedPostingsRe
// no skip data for this term):
private int skipOffset;
+ // docID for next skip point, we won't use skipper if
+ // target docID is not larger than this
+ private int nextSkipDoc;
+
private Bits liveDocs;
public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
@@ -363,6 +367,7 @@ public final class BlockPackedPostingsRe
}
accum = 0;
docUpto = 0;
+ nextSkipDoc = BLOCK_SIZE - 1; // we won't skip if target is found in first block
docBufferUpto = BLOCK_SIZE;
skipped = false;
return this;
@@ -434,7 +439,7 @@ public final class BlockPackedPostingsRe
if (liveDocs == null || liveDocs.get(accum)) {
doc = accum;
- freq = (int) freqBuffer[docBufferUpto];
+ freq = freqBuffer[docBufferUpto];
docBufferUpto++;
if (DEBUG) {
System.out.println(" return doc=" + doc + " freq=" + freq);
@@ -451,10 +456,13 @@ public final class BlockPackedPostingsRe
@Override
public int advance(int target) throws IOException {
// nocommit make frq block load lazy/skippable
+ if (DEBUG) {
+ System.out.println(" FPR.advance target=" + target);
+ }
- // nocommit use skipper!!! it has next last doc id!!
-
- if (docFreq > BLOCK_SIZE && target - accum > BLOCK_SIZE) {
+ // current skip docID < docIDs generated from current buffer <= next skip docID
+ // we don't need to skip if target is buffered already
+ if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {
if (DEBUG) {
System.out.println("load skipper");
@@ -493,6 +501,7 @@ public final class BlockPackedPostingsRe
accum = skipper.getDoc(); // actually, this is just lastSkipEntry
docIn.seek(skipper.getDocPointer()); // now point to the block we want to search
}
+ nextSkipDoc = skipper.getNextSkipDoc();
}
// Now scan... this is an inlined/pared down version
@@ -526,7 +535,7 @@ public final class BlockPackedPostingsRe
if (DEBUG) {
System.out.println(" return doc=" + accum);
}
- freq = (int) freqBuffer[docBufferUpto];
+ freq = freqBuffer[docBufferUpto];
docBufferUpto++;
return doc = accum;
} else {
@@ -597,6 +606,8 @@ public final class BlockPackedPostingsRe
// no skip data for this term):
private int skipOffset;
+ private int nextSkipDoc;
+
private Bits liveDocs;
public BlockDocsAndPositionsEnum(FieldInfo fieldInfo) throws IOException {
@@ -638,6 +649,7 @@ public final class BlockPackedPostingsRe
doc = -1;
accum = 0;
docUpto = 0;
+ nextSkipDoc = BLOCK_SIZE - 1;
docBufferUpto = BLOCK_SIZE;
skipped = false;
return this;
@@ -730,8 +742,8 @@ public final class BlockPackedPostingsRe
if (DEBUG) {
System.out.println(" accum=" + accum + " docDeltaBuffer[" + docBufferUpto + "]=" + docDeltaBuffer[docBufferUpto]);
}
- accum += (int) docDeltaBuffer[docBufferUpto];
- freq = (int) freqBuffer[docBufferUpto];
+ accum += docDeltaBuffer[docBufferUpto];
+ freq = freqBuffer[docBufferUpto];
posPendingCount += freq;
docBufferUpto++;
docUpto++;
@@ -757,11 +769,7 @@ public final class BlockPackedPostingsRe
System.out.println(" FPR.advance target=" + target);
}
- // nocommit 2 is heuristic guess!!
- // nocommit put cheating back! does it help?
- // nocommit use skipper!!! it has next last doc id!!
- //if (docFreq > blockSize && target - (blockSize - docBufferUpto) - 2*blockSize > accum) {
- if (docFreq > BLOCK_SIZE && target - accum > BLOCK_SIZE) {
+ if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {
if (DEBUG) {
System.out.println(" try skipper");
}
@@ -807,6 +815,7 @@ public final class BlockPackedPostingsRe
posPendingFP = skipper.getPosPointer();
posPendingCount = skipper.getPosBufferUpto();
}
+ nextSkipDoc = skipper.getNextSkipDoc();
}
// Now scan... this is an inlined/pared down version
@@ -829,7 +838,7 @@ public final class BlockPackedPostingsRe
refillDocs();
}
accum += docDeltaBuffer[docBufferUpto];
- freq = (int) freqBuffer[docBufferUpto];
+ freq = freqBuffer[docBufferUpto];
posPendingCount += freq;
docBufferUpto++;
docUpto++;
@@ -915,7 +924,7 @@ public final class BlockPackedPostingsRe
refillPositions();
posBufferUpto = 0;
}
- position += (int) posDeltaBuffer[posBufferUpto++];
+ position += posDeltaBuffer[posBufferUpto++];
posPendingCount--;
if (DEBUG) {
System.out.println(" return pos=" + position);
@@ -1020,6 +1029,8 @@ public final class BlockPackedPostingsRe
// no skip data for this term):
private int skipOffset;
+ private int nextSkipDoc;
+
private Bits liveDocs;
public EverythingEnum(FieldInfo fieldInfo) throws IOException {
@@ -1082,6 +1093,7 @@ public final class BlockPackedPostingsRe
doc = -1;
accum = 0;
docUpto = 0;
+ nextSkipDoc = BLOCK_SIZE - 1;
docBufferUpto = BLOCK_SIZE;
skipped = false;
return this;
@@ -1222,7 +1234,7 @@ public final class BlockPackedPostingsRe
System.out.println(" accum=" + accum + " docDeltaBuffer[" + docBufferUpto + "]=" + docDeltaBuffer[docBufferUpto]);
}
accum += docDeltaBuffer[docBufferUpto];
- freq = (int) freqBuffer[docBufferUpto];
+ freq = freqBuffer[docBufferUpto];
posPendingCount += freq;
docBufferUpto++;
docUpto++;
@@ -1251,11 +1263,7 @@ public final class BlockPackedPostingsRe
System.out.println(" FPR.advance target=" + target);
}
- // nocommit 2 is heuristic guess!!
- // nocommit put cheating back! does it help?
- // nocommit use skipper!!! it has next last doc id!!
- //if (docFreq > blockSize && target - (blockSize - docBufferUpto) - 2*blockSize > accum) {
- if (docFreq > BLOCK_SIZE && target - accum > BLOCK_SIZE) {
+ if (docFreq > BLOCK_SIZE && target > nextSkipDoc) {
if (DEBUG) {
System.out.println(" try skipper");
@@ -1305,6 +1313,7 @@ public final class BlockPackedPostingsRe
lastStartOffset = skipper.getStartOffset();
payloadByteUpto = skipper.getPayloadByteUpto();
}
+ nextSkipDoc = skipper.getNextSkipDoc();
}
// nocommit inline nextDoc here
@@ -1451,12 +1460,12 @@ public final class BlockPackedPostingsRe
position += posDeltaBuffer[posBufferUpto];
if (indexHasPayloads) {
- payloadLength = (int) payloadLengthBuffer[posBufferUpto];
+ payloadLength = payloadLengthBuffer[posBufferUpto];
}
if (indexHasOffsets) {
- startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];
- endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];
+ startOffset = lastStartOffset + offsetStartDeltaBuffer[posBufferUpto];
+ endOffset = startOffset + offsetLengthBuffer[posBufferUpto];
lastStartOffset = startOffset;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java?rev=1370850&r1=1370849&r2=1370850&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedPostingsWriter.java Wed Aug 8 17:28:16 2012
@@ -170,6 +170,7 @@ public final class BlockPackedPostingsWr
docDeltaBuffer = new int[MIN_DATA_SIZE];
freqBuffer = new int[MIN_DATA_SIZE];
+ // nocommit should we try skipping every 2/4 blocks...?
skipWriter = new BlockPackedSkipWriter(maxSkipLevels,
BlockPackedPostingsFormat.BLOCK_SIZE,
state.segmentInfo.getDocCount(),
@@ -387,8 +388,8 @@ public final class BlockPackedPostingsWr
// vInt encode the remaining doc deltas and freqs:
for(int i=0;i<docBufferUpto;i++) {
- final int docDelta = (int) docDeltaBuffer[i];
- final int freq = (int) freqBuffer[i];
+ final int docDelta = docDeltaBuffer[i];
+ final int freq = freqBuffer[i];
if (!fieldHasFreqs) {
docOut.writeVInt(docDelta);
} else if (freqBuffer[i] == 1) {
@@ -426,9 +427,9 @@ public final class BlockPackedPostingsWr
int lastPayloadLength = -1;
int payloadBytesReadUpto = 0;
for(int i=0;i<posBufferUpto;i++) {
- final int posDelta = (int) posDeltaBuffer[i];
+ final int posDelta = posDeltaBuffer[i];
if (fieldHasPayloads) {
- final int payloadLength = (int) payloadLengthBuffer[i];
+ final int payloadLength = payloadLengthBuffer[i];
if (payloadLength != lastPayloadLength) {
lastPayloadLength = payloadLength;
posOut.writeVInt((posDelta<<1)|1);
@@ -456,8 +457,8 @@ public final class BlockPackedPostingsWr
if (DEBUG) {
System.out.println(" write offset @ pos.fp=" + posOut.getFilePointer());
}
- posOut.writeVInt((int) offsetStartDeltaBuffer[i]);
- posOut.writeVInt((int) offsetLengthBuffer[i]);
+ posOut.writeVInt(offsetStartDeltaBuffer[i]);
+ posOut.writeVInt(offsetLengthBuffer[i]);
}
}
@@ -475,7 +476,7 @@ public final class BlockPackedPostingsWr
int skipOffset;
if (docCount > BLOCK_SIZE) {
- skipOffset = (int) (skipWriter.writeSkip(docOut)-docTermStartFP);
+ skipOffset = (int) (skipWriter.writeSkip(docOut) - docTermStartFP);
if (DEBUG) {
System.out.println("skip packet " + (docOut.getFilePointer() - (docTermStartFP + skipOffset)) + " bytes");
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java?rev=1370850&r1=1370849&r2=1370850&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/BlockPackedSkipReader.java Wed Aug 8 17:28:16 2012
@@ -151,6 +151,10 @@ final class BlockPackedSkipReader extend
return lastPayloadByteUpto;
}
+ public int getNextSkipDoc() {
+ return skipDoc[0];
+ }
+
@Override
protected void seekChild(int level) throws IOException {
super.seekChild(level);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java?rev=1370850&r1=1370849&r2=1370850&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/blockpacked/ForUtil.java Wed Aug 8 17:28:16 2012
@@ -147,7 +147,7 @@ final class ForUtil {
void writeBlock(int[] data, byte[] encoded, IndexOutput out) throws IOException {
if (isAllEqual(data)) {
out.writeVInt(ALL_VALUES_EQUAL);
- out.writeInt((int) data[0]);
+ out.writeInt(data[0]);
return;
}