You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/10/23 15:10:09 UTC
svn commit: r1401284 - in /lucene/dev/trunk/lucene: ./
core/src/java/org/apache/lucene/codecs/lucene41/
Author: rmuir
Date: Tue Oct 23 13:10:09 2012
New Revision: 1401284
URL: http://svn.apache.org/viewvc?rev=1401284&view=rev
Log:
LUCENE-4498: pulse docFreq=1 in 4.1 codec
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1401284&r1=1401283&r2=1401284&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Oct 23 13:10:09 2012
@@ -108,6 +108,10 @@ Optimizations
* LUCENE-4497: Don't write PosVIntCount to the positions file in
Lucene41PostingsFormat, as its always totalTermFreq % BLOCK_SIZE. (Robert Muir)
+* LUCENE-4498: In Lucene41PostingsFormat, when a term appears in only one document,
+ Instead of writing a file pointer to a VIntBlock containing the doc id, just
+ write the doc id. (Mike McCandless, Robert Muir)
+
Build
* LUCENE-4451: Memory leak per unique thread caused by
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java?rev=1401284&r1=1401283&r2=1401284&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsFormat.java Tue Oct 23 13:10:09 2012
@@ -127,10 +127,10 @@ import org.apache.lucene.util.packed.Pac
*
* <ul>
* <li>Postings Metadata --> Header, PackedBlockSize</li>
- * <li>Term Metadata --> DocFPDelta, PosFPDelta?, PosVIntBlockFPDelta?, PayFPDelta?,
+ * <li>Term Metadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?, PayFPDelta?,
* SkipFPDelta?</li>
* <li>Header, --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>PackedBlockSize --> {@link DataOutput#writeVInt VInt}</li>
+ * <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}</li>
* <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> {@link DataOutput#writeVLong VLong}</li>
* </ul>
* <p>Notes:</p>
@@ -162,6 +162,9 @@ import org.apache.lucene.util.packed.Pac
* file. In particular, it is the length of the TermFreq data.
* SkipDelta is only stored if DocFreq is not smaller than SkipMinimum
* (i.e. 8 in Lucene41PostingsFormat).</li>
+ * <li>SingletonDocID is an optimization when a term only appears in one document. In this case, instead
+ * of writing a file pointer to the .doc file (DocFPDelta), and then a VIntBlock at that location, the
+ * single document ID is written to the term dictionary.</li>
* </ul>
* </dd>
* </dl>
@@ -277,7 +280,7 @@ import org.apache.lucene.util.packed.Pac
* <li>VIntBlock --> <PositionDelta[, PayloadLength?], PayloadData?,
* OffsetDelta?, OffsetLength?><sup>PosVIntCount</sup>
* <li>PackedPosDeltaBlock --> {@link PackedInts PackedInts}</li>
- * <li>PosVIntCount, PositionDelta, OffsetDelta, OffsetLength -->
+ * <li>PositionDelta, OffsetDelta, OffsetLength -->
* {@link DataOutput#writeVInt VInt}</li>
* <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayLength</sup></li>
* </ul>
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1401284&r1=1401283&r2=1401284&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Tue Oct 23 13:10:09 2012
@@ -148,6 +148,9 @@ public final class Lucene41PostingsReade
long payStartFP;
long skipOffset;
long lastPosBlockOffset;
+ // docid when there is a single pulsed posting, otherwise -1
+ // freq is always implicitly totalTermFreq in this case.
+ int singletonDocID;
// Only used by the "primary" TermState -- clones don't
// copy this (basically they are "transient"):
@@ -170,6 +173,7 @@ public final class Lucene41PostingsReade
payStartFP = other.payStartFP;
lastPosBlockOffset = other.lastPosBlockOffset;
skipOffset = other.skipOffset;
+ singletonDocID = other.singletonDocID;
// Do not copy bytes, bytesReader (else TermState is
// very heavy, ie drags around the entire block's
@@ -179,7 +183,7 @@ public final class Lucene41PostingsReade
@Override
public String toString() {
- return super.toString() + " docStartFP=" + docStartFP + " posStartFP=" + posStartFP + " payStartFP=" + payStartFP + " lastPosBlockOffset=" + lastPosBlockOffset;
+ return super.toString() + " docStartFP=" + docStartFP + " posStartFP=" + posStartFP + " payStartFP=" + payStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID;
}
}
@@ -223,7 +227,13 @@ public final class Lucene41PostingsReade
final DataInput in = termState.bytesReader;
if (isFirstTerm) {
- termState.docStartFP = in.readVLong();
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
+ termState.docStartFP = 0;
+ } else {
+ termState.singletonDocID = -1;
+ termState.docStartFP = in.readVLong();
+ }
if (fieldHasPositions) {
termState.posStartFP = in.readVLong();
if (termState.totalTermFreq > BLOCK_SIZE) {
@@ -238,7 +248,12 @@ public final class Lucene41PostingsReade
}
}
} else {
- termState.docStartFP += in.readVLong();
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
+ } else {
+ termState.singletonDocID = -1;
+ termState.docStartFP += in.readVLong();
+ }
if (fieldHasPositions) {
termState.posStartFP += in.readVLong();
if (termState.totalTermFreq > BLOCK_SIZE) {
@@ -327,13 +342,14 @@ public final class Lucene41PostingsReade
final IndexInput startDocIn;
- final IndexInput docIn;
+ IndexInput docIn;
final boolean indexHasFreq;
final boolean indexHasPos;
final boolean indexHasOffsets;
final boolean indexHasPayloads;
private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // sum of freqs in this posting list (or docFreq when omitted)
private int docUpto; // how many docs we've read
private int doc; // doc we last read
private int accum; // accumulator for doc deltas
@@ -354,10 +370,11 @@ public final class Lucene41PostingsReade
private Bits liveDocs;
private boolean needsFreq; // true if the caller actually needs frequencies
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
this.startDocIn = Lucene41PostingsReader.this.docIn;
- this.docIn = startDocIn.clone();
+ this.docIn = null;
indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
indexHasPos = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
@@ -378,9 +395,17 @@ public final class Lucene41PostingsReade
// System.out.println(" FPR.reset: termState=" + termState);
// }
docFreq = termState.docFreq;
+ totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;
docTermStartFP = termState.docStartFP;
- docIn.seek(docTermStartFP);
skipOffset = termState.skipOffset;
+ singletonDocID = termState.singletonDocID;
+ if (docFreq > 1) {
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ docIn.seek(docTermStartFP);
+ }
doc = -1;
this.needsFreq = (flags & DocsEnum.FLAG_FREQS) != 0;
@@ -425,6 +450,9 @@ public final class Lucene41PostingsReade
forUtil.skipBlock(docIn); // skip over freqs
}
}
+ } else if (docFreq == 1) {
+ docDeltaBuffer[0] = singletonDocID;
+ freqBuffer[0] = (int) totalTermFreq;
} else {
// Read vInts:
// if (DEBUG) {
@@ -590,7 +618,7 @@ public final class Lucene41PostingsReade
final IndexInput startDocIn;
- final IndexInput docIn;
+ IndexInput docIn;
final IndexInput posIn;
final boolean indexHasOffsets;
@@ -635,10 +663,11 @@ public final class Lucene41PostingsReade
private int nextSkipDoc;
private Bits liveDocs;
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
public BlockDocsAndPositionsEnum(FieldInfo fieldInfo) throws IOException {
this.startDocIn = Lucene41PostingsReader.this.docIn;
- this.docIn = startDocIn.clone();
+ this.docIn = null;
this.posIn = Lucene41PostingsReader.this.posIn.clone();
encoded = new byte[MAX_ENCODED_SIZE];
indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
@@ -660,9 +689,16 @@ public final class Lucene41PostingsReade
docTermStartFP = termState.docStartFP;
posTermStartFP = termState.posStartFP;
payTermStartFP = termState.payStartFP;
- docIn.seek(docTermStartFP);
skipOffset = termState.skipOffset;
totalTermFreq = termState.totalTermFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docFreq > 1) {
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ docIn.seek(docTermStartFP);
+ }
posPendingFP = posTermStartFP;
posPendingCount = 0;
if (termState.totalTermFreq < BLOCK_SIZE) {
@@ -705,6 +741,9 @@ public final class Lucene41PostingsReade
// System.out.println(" fill freq block from fp=" + docIn.getFilePointer());
// }
forUtil.readBlock(docIn, encoded, freqBuffer);
+ } else if (docFreq == 1) {
+ docDeltaBuffer[0] = singletonDocID;
+ freqBuffer[0] = (int) totalTermFreq;
} else {
// Read vInts:
// if (DEBUG) {
@@ -1002,7 +1041,7 @@ public final class Lucene41PostingsReade
final IndexInput startDocIn;
- final IndexInput docIn;
+ IndexInput docIn;
final IndexInput posIn;
final IndexInput payIn;
final BytesRef payload;
@@ -1056,10 +1095,11 @@ public final class Lucene41PostingsReade
private boolean needsOffsets; // true if we actually need offsets
private boolean needsPayloads; // true if we actually need payloads
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
public EverythingEnum(FieldInfo fieldInfo) throws IOException {
this.startDocIn = Lucene41PostingsReader.this.docIn;
- this.docIn = startDocIn.clone();
+ this.docIn = null;
this.posIn = Lucene41PostingsReader.this.posIn.clone();
this.payIn = Lucene41PostingsReader.this.payIn.clone();
encoded = new byte[MAX_ENCODED_SIZE];
@@ -1101,9 +1141,16 @@ public final class Lucene41PostingsReade
docTermStartFP = termState.docStartFP;
posTermStartFP = termState.posStartFP;
payTermStartFP = termState.payStartFP;
- docIn.seek(docTermStartFP);
skipOffset = termState.skipOffset;
totalTermFreq = termState.totalTermFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docFreq > 1) {
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ docIn.seek(docTermStartFP);
+ }
posPendingFP = posTermStartFP;
payPendingFP = payTermStartFP;
posPendingCount = 0;
@@ -1150,6 +1197,9 @@ public final class Lucene41PostingsReade
// System.out.println(" fill freq block from fp=" + docIn.getFilePointer());
// }
forUtil.readBlock(docIn, encoded, freqBuffer);
+ } else if (docFreq == 1) {
+ docDeltaBuffer[0] = singletonDocID;
+ freqBuffer[0] = (int) totalTermFreq;
} else {
// if (DEBUG) {
// System.out.println(" fill last vInt doc block from fp=" + docIn.getFilePointer());
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java?rev=1401284&r1=1401283&r2=1401284&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java Tue Oct 23 13:10:09 2012
@@ -354,13 +354,15 @@ public final class Lucene41PostingsWrite
public final long payStartFP;
public final long skipOffset;
public final long lastPosBlockOffset;
+ public final int singletonDocID;
- public PendingTerm(long docStartFP, long posStartFP, long payStartFP, long skipOffset, long lastPosBlockOffset) {
+ public PendingTerm(long docStartFP, long posStartFP, long payStartFP, long skipOffset, long lastPosBlockOffset, int singletonDocID) {
this.docStartFP = docStartFP;
this.posStartFP = posStartFP;
this.payStartFP = payStartFP;
this.skipOffset = skipOffset;
this.lastPosBlockOffset = lastPosBlockOffset;
+ this.singletonDocID = singletonDocID;
}
}
@@ -384,18 +386,26 @@ public final class Lucene41PostingsWrite
// System.out.println(" write doc/freq vInt block (count=" + docBufferUpto + ") at fp=" + docOut.getFilePointer() + " docTermStartFP=" + docTermStartFP);
// }
// }
-
- // vInt encode the remaining doc deltas and freqs:
- for(int i=0;i<docBufferUpto;i++) {
- final int docDelta = docDeltaBuffer[i];
- final int freq = freqBuffer[i];
- if (!fieldHasFreqs) {
- docOut.writeVInt(docDelta);
- } else if (freqBuffer[i] == 1) {
- docOut.writeVInt((docDelta<<1)|1);
- } else {
- docOut.writeVInt(docDelta<<1);
- docOut.writeVInt(freq);
+
+ // docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to it.
+ final int singletonDocID;
+ if (stats.docFreq == 1) {
+ // pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
+ singletonDocID = docDeltaBuffer[0];
+ } else {
+ singletonDocID = -1;
+ // vInt encode the remaining doc deltas and freqs:
+ for(int i=0;i<docBufferUpto;i++) {
+ final int docDelta = docDeltaBuffer[i];
+ final int freq = freqBuffer[i];
+ if (!fieldHasFreqs) {
+ docOut.writeVInt(docDelta);
+ } else if (freqBuffer[i] == 1) {
+ docOut.writeVInt((docDelta<<1)|1);
+ } else {
+ docOut.writeVInt(docDelta<<1);
+ docOut.writeVInt(freq);
+ }
}
}
@@ -507,7 +517,7 @@ public final class Lucene41PostingsWrite
// System.out.println(" payStartFP=" + payStartFP);
// }
- pendingTerms.add(new PendingTerm(docTermStartFP, posTermStartFP, payStartFP, skipOffset, lastPosBlockOffset));
+ pendingTerms.add(new PendingTerm(docTermStartFP, posTermStartFP, payStartFP, skipOffset, lastPosBlockOffset, singletonDocID));
docBufferUpto = 0;
posBufferUpto = 0;
lastDocID = 0;
@@ -535,8 +545,12 @@ public final class Lucene41PostingsWrite
for(int idx=limit-count; idx<limit; idx++) {
PendingTerm term = pendingTerms.get(idx);
- bytesWriter.writeVLong(term.docStartFP - lastDocStartFP);
- lastDocStartFP = term.docStartFP;
+ if (term.singletonDocID == -1) {
+ bytesWriter.writeVLong(term.docStartFP - lastDocStartFP);
+ lastDocStartFP = term.docStartFP;
+ } else {
+ bytesWriter.writeVInt(term.singletonDocID);
+ }
if (fieldHasPositions) {
bytesWriter.writeVLong(term.posStartFP - lastPosStartFP);