You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ha...@apache.org on 2013/08/23 16:34:47 UTC
svn commit: r1516860 [2/2] - in /lucene/dev/branches/lucene3069/lucene:
codecs/src/java/org/apache/lucene/codecs/blockterms/
codecs/src/java/org/apache/lucene/codecs/pulsing/
codecs/src/java/org/apache/lucene/codecs/sep/
core/src/java/org/apache/lucene...
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java?rev=1516860&r1=1516859&r2=1516860&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsReader.java Fri Aug 23 14:34:47 2013
@@ -152,11 +152,6 @@ public final class Lucene41PostingsReade
// freq is always implicitly totalTermFreq in this case.
int singletonDocID;
- // Only used by the "primary" TermState -- clones don't
- // copy this (basically they are "transient"):
- ByteArrayDataInput bytesReader; // TODO: should this NOT be in the TermState...?
- byte[] bytes;
-
@Override
public IntBlockTermState clone() {
IntBlockTermState other = new IntBlockTermState();
@@ -174,11 +169,6 @@ public final class Lucene41PostingsReade
lastPosBlockOffset = other.lastPosBlockOffset;
skipOffset = other.skipOffset;
singletonDocID = other.singletonDocID;
-
- // Do not copy bytes, bytesReader (else TermState is
- // very heavy, ie drags around the entire block's
- // byte[]). On seek back, if next() is in fact used
- // (rare!), they will be re-read from disk.
}
@Override
@@ -197,78 +187,37 @@ public final class Lucene41PostingsReade
IOUtils.close(docIn, posIn, payIn);
}
- /* Reads but does not decode the byte[] blob holding
- metadata for the current terms block */
- @Override
- public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, BlockTermState _termState) throws IOException {
- final IntBlockTermState termState = (IntBlockTermState) _termState;
-
- final int numBytes = termsIn.readVInt();
-
- if (termState.bytes == null) {
- termState.bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- termState.bytesReader = new ByteArrayDataInput();
- } else if (termState.bytes.length < numBytes) {
- termState.bytes = new byte[ArrayUtil.oversize(numBytes, 1)];
- }
-
- termsIn.readBytes(termState.bytes, 0, numBytes);
- termState.bytesReader.reset(termState.bytes, 0, numBytes);
- }
-
@Override
- public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState)
+ public void decodeTerm(long[] longs, DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
throws IOException {
final IntBlockTermState termState = (IntBlockTermState) _termState;
- final boolean isFirstTerm = termState.termBlockOrd == 0;
final boolean fieldHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
final boolean fieldHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
final boolean fieldHasPayloads = fieldInfo.hasPayloads();
- final DataInput in = termState.bytesReader;
- if (isFirstTerm) {
- if (termState.docFreq == 1) {
- termState.singletonDocID = in.readVInt();
- termState.docStartFP = 0;
- } else {
- termState.singletonDocID = -1;
- termState.docStartFP = in.readVLong();
- }
- if (fieldHasPositions) {
- termState.posStartFP = in.readVLong();
- if (termState.totalTermFreq > BLOCK_SIZE) {
- termState.lastPosBlockOffset = in.readVLong();
- } else {
- termState.lastPosBlockOffset = -1;
- }
- if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
- termState.payStartFP = in.readVLong();
- } else {
- termState.payStartFP = -1;
- }
+ // nocommit: use old version
+ if (absolute) {
+ termState.docStartFP = 0;
+ termState.posStartFP = 0;
+ termState.payStartFP = 0;
+ }
+ termState.docStartFP += longs[0];
+ if (fieldHasPositions) {
+ termState.posStartFP += longs[1];
+ if (fieldHasOffsets || fieldHasPayloads) {
+ termState.payStartFP += longs[2];
}
+ }
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
} else {
- if (termState.docFreq == 1) {
- termState.singletonDocID = in.readVInt();
+ termState.singletonDocID = -1;
+ }
+ if (fieldHasPositions) {
+ if (termState.totalTermFreq > BLOCK_SIZE) {
+ termState.lastPosBlockOffset = in.readVLong();
} else {
- termState.singletonDocID = -1;
- termState.docStartFP += in.readVLong();
- }
- if (fieldHasPositions) {
- termState.posStartFP += in.readVLong();
- if (termState.totalTermFreq > BLOCK_SIZE) {
- termState.lastPosBlockOffset = in.readVLong();
- } else {
- termState.lastPosBlockOffset = -1;
- }
- if ((fieldHasPayloads || fieldHasOffsets) && termState.totalTermFreq >= BLOCK_SIZE) {
- long delta = in.readVLong();
- if (termState.payStartFP == -1) {
- termState.payStartFP = delta;
- } else {
- termState.payStartFP += delta;
- }
- }
+ termState.lastPosBlockOffset = -1;
}
}
Modified: lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java?rev=1516860&r1=1516859&r2=1516860&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java Fri Aug 23 14:34:47 2013
@@ -25,14 +25,15 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
@@ -71,7 +72,8 @@ public final class Lucene41PostingsWrite
final IndexOutput posOut;
final IndexOutput payOut;
- private IndexOutput termsOut;
+ final static IntBlockTermState emptyState = new IntBlockTermState();
+ IntBlockTermState lastState;
// How current field indexes postings:
private boolean fieldHasFreqs;
@@ -79,7 +81,7 @@ public final class Lucene41PostingsWrite
private boolean fieldHasOffsets;
private boolean fieldHasPayloads;
- // Holds starting file pointers for each term:
+ // Holds starting file pointers for current term:
private long docTermStartFP;
private long posTermStartFP;
private long payTermStartFP;
@@ -188,21 +190,50 @@ public final class Lucene41PostingsWrite
this(state, PackedInts.COMPACT);
}
+ private final static class IntBlockTermState extends BlockTermState {
+ long docTermStartFP = 0;
+ long posTermStartFP = 0;
+ long payTermStartFP = 0;
+ long skipOffset = -1;
+ long lastPosBlockOffset = -1;
+ int singletonDocID = -1;
+ @Override
+ public String toString() {
+ return super.toString() + " docStartFP=" + docTermStartFP + " posStartFP=" + posTermStartFP + " payStartFP=" + payTermStartFP + " lastPosBlockOffset=" + lastPosBlockOffset + " singletonDocID=" + singletonDocID;
+ }
+ }
+
+ @Override
+ public IntBlockTermState newTermState() {
+ return new IntBlockTermState();
+ }
+
@Override
- public void start(IndexOutput termsOut) throws IOException {
- this.termsOut = termsOut;
+ public void init(IndexOutput termsOut) throws IOException {
CodecUtil.writeHeader(termsOut, TERMS_CODEC, VERSION_CURRENT);
termsOut.writeVInt(BLOCK_SIZE);
}
+ // nocommit better name?
+
@Override
- public void setField(FieldInfo fieldInfo) {
+ public int setField(FieldInfo fieldInfo) {
IndexOptions indexOptions = fieldInfo.getIndexOptions();
fieldHasFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
fieldHasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
fieldHasPayloads = fieldInfo.hasPayloads();
skipWriter.setField(fieldHasPositions, fieldHasOffsets, fieldHasPayloads);
+ lastState = emptyState;
+ if (fieldHasPositions) {
+ if (fieldHasPayloads || fieldHasOffsets) {
+ return 3; // doc + pos + pay FP
+ } else {
+ return 2; // doc + pos FP
+ }
+ } else {
+ return 1; // doc FP
+ }
}
@Override
@@ -348,37 +379,18 @@ public final class Lucene41PostingsWrite
}
}
- private static class PendingTerm {
- public final long docStartFP;
- public final long posStartFP;
- public final long payStartFP;
- public final long skipOffset;
- public final long lastPosBlockOffset;
- public final int singletonDocID;
-
- public PendingTerm(long docStartFP, long posStartFP, long payStartFP, long skipOffset, long lastPosBlockOffset, int singletonDocID) {
- this.docStartFP = docStartFP;
- this.posStartFP = posStartFP;
- this.payStartFP = payStartFP;
- this.skipOffset = skipOffset;
- this.lastPosBlockOffset = lastPosBlockOffset;
- this.singletonDocID = singletonDocID;
- }
- }
-
- private final List<PendingTerm> pendingTerms = new ArrayList<PendingTerm>();
-
/** Called when we are done adding docs to this term */
@Override
- public void finishTerm(TermStats stats) throws IOException {
- assert stats.docFreq > 0;
+ public void finishTerm(BlockTermState _state) throws IOException {
+ IntBlockTermState state = (IntBlockTermState) _state;
+ assert state.docFreq > 0;
// TODO: wasteful we are counting this (counting # docs
// for this term) in two places?
- assert stats.docFreq == docCount: stats.docFreq + " vs " + docCount;
+ assert state.docFreq == docCount: state.docFreq + " vs " + docCount;
// if (DEBUG) {
- // System.out.println("FPW.finishTerm docFreq=" + stats.docFreq);
+ // System.out.println("FPW.finishTerm docFreq=" + state.docFreq);
// }
// if (DEBUG) {
@@ -389,7 +401,7 @@ public final class Lucene41PostingsWrite
// docFreq == 1, don't write the single docid/freq to a separate file along with a pointer to it.
final int singletonDocID;
- if (stats.docFreq == 1) {
+ if (state.docFreq == 1) {
// pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
singletonDocID = docDeltaBuffer[0];
} else {
@@ -420,8 +432,8 @@ public final class Lucene41PostingsWrite
// totalTermFreq is just total number of positions(or payloads, or offsets)
// associated with current term.
- assert stats.totalTermFreq != -1;
- if (stats.totalTermFreq > BLOCK_SIZE) {
+ assert state.totalTermFreq != -1;
+ if (state.totalTermFreq > BLOCK_SIZE) {
// record file offset for last pos in last block
lastPosBlockOffset = posOut.getFilePointer() - posTermStartFP;
} else {
@@ -486,7 +498,7 @@ public final class Lucene41PostingsWrite
}
}
// if (DEBUG) {
- // System.out.println(" totalTermFreq=" + stats.totalTermFreq + " lastPosBlockOffset=" + lastPosBlockOffset);
+ // System.out.println(" totalTermFreq=" + state.totalTermFreq + " lastPosBlockOffset=" + lastPosBlockOffset);
// }
} else {
lastPosBlockOffset = -1;
@@ -505,76 +517,48 @@ public final class Lucene41PostingsWrite
// System.out.println(" no skip: docCount=" + docCount);
// }
}
-
- long payStartFP;
- if (stats.totalTermFreq >= BLOCK_SIZE) {
- payStartFP = payTermStartFP;
- } else {
- payStartFP = -1;
- }
-
// if (DEBUG) {
// System.out.println(" payStartFP=" + payStartFP);
// }
-
- pendingTerms.add(new PendingTerm(docTermStartFP, posTermStartFP, payStartFP, skipOffset, lastPosBlockOffset, singletonDocID));
+ state.docTermStartFP = docTermStartFP;
+ state.posTermStartFP = posTermStartFP;
+ state.payTermStartFP = payTermStartFP;
+ state.singletonDocID = singletonDocID;
+ state.skipOffset = skipOffset;
+ state.lastPosBlockOffset = lastPosBlockOffset;
docBufferUpto = 0;
posBufferUpto = 0;
lastDocID = 0;
docCount = 0;
}
-
- private final RAMOutputStream bytesWriter = new RAMOutputStream();
+
+ // nocommit explain about the "don't care" values
@Override
- public void flushTermsBlock(int start, int count) throws IOException {
-
- if (count == 0) {
- termsOut.writeByte((byte) 0);
- return;
+ public void encodeTerm(long[] longs, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
+ IntBlockTermState state = (IntBlockTermState)_state;
+ if (absolute) {
+ lastState = emptyState;
}
-
- assert start <= pendingTerms.size();
- assert count <= start;
-
- final int limit = pendingTerms.size() - start + count;
-
- long lastDocStartFP = 0;
- long lastPosStartFP = 0;
- long lastPayStartFP = 0;
- for(int idx=limit-count; idx<limit; idx++) {
- PendingTerm term = pendingTerms.get(idx);
-
- if (term.singletonDocID == -1) {
- bytesWriter.writeVLong(term.docStartFP - lastDocStartFP);
- lastDocStartFP = term.docStartFP;
- } else {
- bytesWriter.writeVInt(term.singletonDocID);
- }
-
- if (fieldHasPositions) {
- bytesWriter.writeVLong(term.posStartFP - lastPosStartFP);
- lastPosStartFP = term.posStartFP;
- if (term.lastPosBlockOffset != -1) {
- bytesWriter.writeVLong(term.lastPosBlockOffset);
- }
- if ((fieldHasPayloads || fieldHasOffsets) && term.payStartFP != -1) {
- bytesWriter.writeVLong(term.payStartFP - lastPayStartFP);
- lastPayStartFP = term.payStartFP;
- }
+ longs[0] = state.docTermStartFP - lastState.docTermStartFP;
+ if (fieldHasPositions) {
+ longs[1] = state.posTermStartFP - lastState.posTermStartFP;
+ if (fieldHasPayloads || fieldHasOffsets) {
+ longs[2] = state.payTermStartFP - lastState.payTermStartFP;
}
-
- if (term.skipOffset != -1) {
- bytesWriter.writeVLong(term.skipOffset);
+ }
+ if (state.singletonDocID != -1) {
+ out.writeVInt(state.singletonDocID);
+ }
+ if (fieldHasPositions) {
+ if (state.lastPosBlockOffset != -1) {
+ out.writeVLong(state.lastPosBlockOffset);
}
}
-
- termsOut.writeVInt((int) bytesWriter.getFilePointer());
- bytesWriter.writeTo(termsOut);
- bytesWriter.reset();
-
- // Remove the terms we just wrote:
- pendingTerms.subList(limit-count, limit).clear();
+ if (state.skipOffset != -1) {
+ out.writeVLong(state.skipOffset);
+ }
+ lastState = state;
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java?rev=1516860&r1=1516859&r2=1516860&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java Fri Aug 23 14:34:47 2013
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.TermStats;
@@ -33,6 +34,7 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.BytesRef;
@@ -67,7 +69,6 @@ public final class Lucene40PostingsWrite
*/
final int maxSkipLevels = 10;
final int totalNumDocs;
- IndexOutput termsOut;
IndexOptions indexOptions;
boolean storePayloads;
@@ -81,6 +82,9 @@ public final class Lucene40PostingsWrite
int lastPosition;
int lastOffset;
+ final static StandardTermState emptyState = new StandardTermState();
+ StandardTermState lastState;
+
// private String segment;
/** Creates a {@link Lucene40PostingsWriter}, with the
@@ -134,8 +138,7 @@ public final class Lucene40PostingsWrite
}
@Override
- public void start(IndexOutput termsOut) throws IOException {
- this.termsOut = termsOut;
+ public void init(IndexOutput termsOut) throws IOException {
CodecUtil.writeHeader(termsOut, Lucene40PostingsReader.TERMS_CODEC, Lucene40PostingsReader.VERSION_CURRENT);
termsOut.writeInt(skipInterval); // write skipInterval
termsOut.writeInt(maxSkipLevels); // write maxSkipLevels
@@ -143,6 +146,12 @@ public final class Lucene40PostingsWrite
}
@Override
+ public BlockTermState newTermState() {
+ return new StandardTermState();
+ }
+
+
+ @Override
public void startTerm() {
freqStart = freqOut.getFilePointer();
//if (DEBUG) System.out.println("SPW: startTerm freqOut.fp=" + freqStart);
@@ -159,7 +168,7 @@ public final class Lucene40PostingsWrite
// Currently, this instance is re-used across fields, so
// our parent calls setField whenever the field changes
@Override
- public void setField(FieldInfo fieldInfo) {
+ public int setField(FieldInfo fieldInfo) {
//System.out.println("SPW: setField");
/*
if (BlockTreeTermsWriter.DEBUG && fieldInfo.name.equals("id")) {
@@ -173,8 +182,10 @@ public final class Lucene40PostingsWrite
storeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
storePayloads = fieldInfo.hasPayloads();
+ lastState = emptyState;
//System.out.println(" set init blockFreqStart=" + freqStart);
//System.out.println(" set init blockProxStart=" + proxStart);
+ return 0;
}
int lastDocID;
@@ -265,94 +276,48 @@ public final class Lucene40PostingsWrite
public void finishDoc() {
}
- private static class PendingTerm {
- public final long freqStart;
- public final long proxStart;
- public final long skipOffset;
-
- public PendingTerm(long freqStart, long proxStart, long skipOffset) {
- this.freqStart = freqStart;
- this.proxStart = proxStart;
- this.skipOffset = skipOffset;
- }
+ private static class StandardTermState extends BlockTermState {
+ public long freqStart;
+ public long proxStart;
+ public long skipOffset;
}
- private final List<PendingTerm> pendingTerms = new ArrayList<PendingTerm>();
-
/** Called when we are done adding docs to this term */
@Override
- public void finishTerm(TermStats stats) throws IOException {
-
+ public void finishTerm(BlockTermState _state) throws IOException {
+ StandardTermState state = (StandardTermState)_state;
// if (DEBUG) System.out.println("SPW: finishTerm seg=" + segment + " freqStart=" + freqStart);
- assert stats.docFreq > 0;
+ assert state.docFreq > 0;
// TODO: wasteful we are counting this (counting # docs
// for this term) in two places?
- assert stats.docFreq == df;
-
- final long skipOffset;
+ assert state.docFreq == df;
+ state.freqStart = freqStart;
+ state.proxStart = proxStart;
if (df >= skipMinimum) {
- skipOffset = skipListWriter.writeSkip(freqOut)-freqStart;
+ state.skipOffset = skipListWriter.writeSkip(freqOut)-freqStart;
} else {
- skipOffset = -1;
+ state.skipOffset = -1;
}
-
- pendingTerms.add(new PendingTerm(freqStart, proxStart, skipOffset));
-
lastDocID = 0;
df = 0;
}
- private final RAMOutputStream bytesWriter = new RAMOutputStream();
-
@Override
- public void flushTermsBlock(int start, int count) throws IOException {
- //if (DEBUG) System.out.println("SPW: flushTermsBlock start=" + start + " count=" + count + " left=" + (pendingTerms.size()-count) + " pendingTerms.size()=" + pendingTerms.size());
-
- if (count == 0) {
- termsOut.writeByte((byte) 0);
- return;
- }
-
- assert start <= pendingTerms.size();
- assert count <= start;
-
- final int limit = pendingTerms.size() - start + count;
- final PendingTerm firstTerm = pendingTerms.get(limit - count);
- // First term in block is abs coded:
- bytesWriter.writeVLong(firstTerm.freqStart);
-
- if (firstTerm.skipOffset != -1) {
- assert firstTerm.skipOffset > 0;
- bytesWriter.writeVLong(firstTerm.skipOffset);
+ public void encodeTerm(long[] empty, DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
+ StandardTermState state = (StandardTermState)_state;
+ if (absolute) {
+ lastState = emptyState;
+ }
+ out.writeVLong(state.freqStart - lastState.freqStart);
+ if (state.skipOffset != -1) {
+ assert state.skipOffset > 0;
+ out.writeVLong(state.skipOffset);
}
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
- bytesWriter.writeVLong(firstTerm.proxStart);
+ out.writeVLong(state.proxStart - lastState.proxStart);
}
- long lastFreqStart = firstTerm.freqStart;
- long lastProxStart = firstTerm.proxStart;
- for(int idx=limit-count+1; idx<limit; idx++) {
- final PendingTerm term = pendingTerms.get(idx);
- //if (DEBUG) System.out.println(" write term freqStart=" + term.freqStart);
- // The rest of the terms term are delta coded:
- bytesWriter.writeVLong(term.freqStart - lastFreqStart);
- lastFreqStart = term.freqStart;
- if (term.skipOffset != -1) {
- assert term.skipOffset > 0;
- bytesWriter.writeVLong(term.skipOffset);
- }
- if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0) {
- bytesWriter.writeVLong(term.proxStart - lastProxStart);
- lastProxStart = term.proxStart;
- }
- }
-
- termsOut.writeVInt((int) bytesWriter.getFilePointer());
- bytesWriter.writeTo(termsOut);
- bytesWriter.reset();
-
- // Remove the terms we just wrote:
- pendingTerms.subList(limit-count, limit).clear();
+ lastState = state;
}
@Override
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java?rev=1516860&r1=1516859&r2=1516860&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/mockrandom/MockRandomPostingsFormat.java Fri Aug 23 14:34:47 2013
@@ -183,12 +183,37 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff);
}
- postingsWriter = new PulsingPostingsWriter(totTFCutoff, postingsWriter);
+ postingsWriter = new PulsingPostingsWriter(state, totTFCutoff, postingsWriter);
}
final FieldsConsumer fields;
+ final int t1 = random.nextInt(2);
- if (random.nextBoolean()) {
+ /*
+ final int t1 = random.nextInt(4);
+ if (t1 == 0) {
+ boolean success = false;
+ try {
+ fields = new TempFSTTermsWriter(state, postingsWriter);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsWriter.close();
+ }
+ }
+ } else if (t1 == 1) {
+ boolean success = false;
+ try {
+ fields = new TempFSTOrdTermsWriter(state, postingsWriter);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsWriter.close();
+ }
+ }
+ } else if (t1 == 2) {
+ */
+ if (t1 == 0) {
// Use BlockTree terms dict
if (LuceneTestCase.VERBOSE) {
@@ -322,12 +347,36 @@ public final class MockRandomPostingsFor
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading pulsing postings with totTFCutoff=" + totTFCutoff);
}
- postingsReader = new PulsingPostingsReader(postingsReader);
+ postingsReader = new PulsingPostingsReader(state, postingsReader);
}
final FieldsProducer fields;
-
- if (random.nextBoolean()) {
+ final int t1 = random.nextInt(2);
+ /*
+ final int t1 = random.nextInt(4);
+ if (t1 == 0) {
+ boolean success = false;
+ try {
+ fields = new TempFSTTermsReader(state, postingsReader);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsReader.close();
+ }
+ }
+ } else if (t1 == 1) {
+ boolean success = false;
+ try {
+ fields = new TempFSTOrdTermsReader(state, postingsReader);
+ success = true;
+ } finally {
+ if (!success) {
+ postingsReader.close();
+ }
+ }
+ } else if (t1 == 2) {
+ */
+ if (t1 == 0) {
// Use BlockTree terms dict
if (LuceneTestCase.VERBOSE) {
System.out.println("MockRandomCodec: reading BlockTree terms dict");
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java?rev=1516860&r1=1516859&r2=1516860&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/codecs/nestedpulsing/NestedPulsingPostingsFormat.java Fri Aug 23 14:34:47 2013
@@ -57,8 +57,8 @@ public final class NestedPulsingPostings
try {
docsWriter = new Lucene41PostingsWriter(state);
- pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter);
- pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner);
+ pulsingWriterInner = new PulsingPostingsWriter(state, 2, docsWriter);
+ pulsingWriter = new PulsingPostingsWriter(state, 1, pulsingWriterInner);
FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter,
BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
success = true;
@@ -78,8 +78,8 @@ public final class NestedPulsingPostings
boolean success = false;
try {
docsReader = new Lucene41PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
- pulsingReaderInner = new PulsingPostingsReader(docsReader);
- pulsingReader = new PulsingPostingsReader(pulsingReaderInner);
+ pulsingReaderInner = new PulsingPostingsReader(state, docsReader);
+ pulsingReader = new PulsingPostingsReader(state, pulsingReaderInner);
FieldsProducer ret = new BlockTreeTermsReader(
state.directory, state.fieldInfos, state.segmentInfo,
pulsingReader,
Modified: lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java?rev=1516860&r1=1516859&r2=1516860&view=diff
==============================================================================
--- lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java (original)
+++ lucene/dev/branches/lucene3069/lucene/test-framework/src/java/org/apache/lucene/util/TestRuleSetupAndRestoreClassEnv.java Fri Aug 23 14:34:47 2013
@@ -169,7 +169,7 @@ final class TestRuleSetupAndRestoreClass
final PostingsFormat format;
if ("random".equals(TEST_POSTINGSFORMAT)) {
format = PostingsFormat.forName("Lucene41");
- } else if ("TempRandom".equals(TEST_POSTINGSFORMAT)) {
+ } else if ("MockRandom".equals(TEST_POSTINGSFORMAT)) {
format = new MockRandomPostingsFormat(new Random(random.nextLong()));
} else {
format = PostingsFormat.forName(TEST_POSTINGSFORMAT);