You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/08/21 14:14:42 UTC
svn commit: r1375495 [1/3] - in /lucene/dev/branches/branch_4x: ./
dev-tools/ lucene/ lucene/analysis/ lucene/benchmark/ lucene/core/
lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/codecs/block/ lucene/core/src/ja...
Author: mikemccand
Date: Tue Aug 21 12:14:39 2012
New Revision: 1375495
URL: http://svn.apache.org/viewvc?rev=1375495&view=rev
Log:
LUCENE-3892: merge BlockPostingsFormat from branch
Added:
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/block/
- copied from r1375486, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/block/
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/codecs/block/
- copied from r1375486, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/codecs/block/
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/dev-tools/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/MIGRATE.txt (props changed)
lucene/dev/branches/branch_4x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_4x/lucene/benchmark/ (props changed)
lucene/dev/branches/branch_4x/lucene/build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/common-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepSkipListReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/BulkOperation.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/Packed64SingleBlock.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/PackedWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/packed/gen_BulkOperation.py
lucene/dev/branches/branch_4x/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/packed/TestPackedInts.java
lucene/dev/branches/branch_4x/lucene/demo/ (props changed)
lucene/dev/branches/branch_4x/lucene/facet/ (props changed)
lucene/dev/branches/branch_4x/lucene/grouping/ (props changed)
lucene/dev/branches/branch_4x/lucene/highlighter/ (props changed)
lucene/dev/branches/branch_4x/lucene/ivy-settings.xml (props changed)
lucene/dev/branches/branch_4x/lucene/join/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/misc/ (props changed)
lucene/dev/branches/branch_4x/lucene/module-build.xml (props changed)
lucene/dev/branches/branch_4x/lucene/queries/ (props changed)
lucene/dev/branches/branch_4x/lucene/queryparser/ (props changed)
lucene/dev/branches/branch_4x/lucene/sandbox/ (props changed)
lucene/dev/branches/branch_4x/lucene/spatial/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/mocksep/MockSingleIntIndexInput.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_4x/lucene/tools/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (props changed)
lucene/dev/branches/branch_4x/solr/NOTICE.txt (props changed)
lucene/dev/branches/branch_4x/solr/build.xml (props changed)
lucene/dev/branches/branch_4x/solr/cloud-dev/ (props changed)
lucene/dev/branches/branch_4x/solr/common-build.xml (props changed)
lucene/dev/branches/branch_4x/solr/contrib/ (props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/example/ (props changed)
lucene/dev/branches/branch_4x/solr/lib/ (props changed)
lucene/dev/branches/branch_4x/solr/licenses/ (props changed)
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
lucene/dev/branches/branch_4x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_4x/solr/testlogging.properties (props changed)
lucene/dev/branches/branch_4x/solr/webapp/ (props changed)
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue Aug 21 12:14:39 2012
@@ -15,6 +15,11 @@ New Features
that you must store term vector positions to store payloads.
(Robert Muir)
+* LUCENE-3892: Add a new BlockPostingsFormat that bulk-encodes docs,
+ freqs and positions in large (size 128) packed-int blocks for faster
+ search performance. This was from Han Jiang's 2012 Google Summer of
+ Code project (Han Jiang, Adrien Grand, Robert Muir, Mike McCandless)
+
API Changes
* LUCENE-4299: Added Terms.hasPositions() and Terms.hasOffsets().
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsWriter.java Tue Aug 21 12:14:39 2012
@@ -724,7 +724,7 @@ public class BlockTreeTermsWriter extend
// Write term stats, to separate byte[] blob:
bytesWriter2.writeVInt(term.stats.docFreq);
if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_ONLY) {
- assert term.stats.totalTermFreq >= term.stats.docFreq;
+ assert term.stats.totalTermFreq >= term.stats.docFreq: term.stats.totalTermFreq + " vs " + term.stats.docFreq;
bytesWriter2.writeVLong(term.stats.totalTermFreq - term.stats.docFreq);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java Tue Aug 21 12:14:39 2012
@@ -59,30 +59,36 @@ public abstract class MultiLevelSkipList
private int skipInterval[]; // skipInterval of each level
private int[] numSkipped; // number of docs skipped per level
- private int[] skipDoc; // doc id of current skip entry per level
+ protected int[] skipDoc; // doc id of current skip entry per level
private int lastDoc; // doc id of last read skip entry with docId <= target
private long[] childPointer; // child pointer of current skip entry per level
private long lastChildPointer; // childPointer of last read skip entry with docId <= target
private boolean inputIsBuffered;
-
- public MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) {
+ private final int skipMultiplier;
+
+ protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval, int skipMultiplier) {
this.skipStream = new IndexInput[maxSkipLevels];
this.skipPointer = new long[maxSkipLevels];
this.childPointer = new long[maxSkipLevels];
this.numSkipped = new int[maxSkipLevels];
this.maxNumberOfSkipLevels = maxSkipLevels;
this.skipInterval = new int[maxSkipLevels];
+ this.skipMultiplier = skipMultiplier;
this.skipStream [0]= skipStream;
this.inputIsBuffered = (skipStream instanceof BufferedIndexInput);
this.skipInterval[0] = skipInterval;
for (int i = 1; i < maxSkipLevels; i++) {
// cache skip intervals
- this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval;
+ this.skipInterval[i] = this.skipInterval[i - 1] * skipMultiplier;
}
skipDoc = new int[maxSkipLevels];
}
+ // skipMultiplier and skipInterval are the same:
+ protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) {
+ this(skipStream, maxSkipLevels, skipInterval, skipInterval);
+ }
/** Returns the id of the doc to which the last call of {@link #skipTo(int)}
* has skipped. */
@@ -157,7 +163,7 @@ public abstract class MultiLevelSkipList
numSkipped[level] = numSkipped[level + 1] - skipInterval[level + 1];
skipDoc[level] = lastDoc;
if (level > 0) {
- childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1];
+ childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1];
}
}
@@ -187,7 +193,12 @@ public abstract class MultiLevelSkipList
/** Loads the skip levels */
private void loadSkipLevels() throws IOException {
- numberOfSkipLevels = MathUtil.log(docCount, skipInterval[0]);
+ if (docCount <= skipInterval[0]) {
+ numberOfSkipLevels = 1;
+ } else {
+ numberOfSkipLevels = 1+MathUtil.log(docCount/skipInterval[0], skipMultiplier);
+ }
+
if (numberOfSkipLevels > maxNumberOfSkipLevels) {
numberOfSkipLevels = maxNumberOfSkipLevels;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java Tue Aug 21 12:14:39 2012
@@ -26,6 +26,8 @@ import org.apache.lucene.util.MathUtil;
/**
* This abstract class writes skip lists with multiple levels.
*
+ * <pre>
+ *
* Example for skipInterval = 3:
* c (skip level 2)
* c c c (skip level 1)
@@ -45,6 +47,7 @@ import org.apache.lucene.util.MathUtil;
*
* While this class takes care of writing the different skip levels,
* subclasses must define the actual format of the skip data.
+ * </pre>
* @lucene.experimental
*/
@@ -54,15 +57,23 @@ public abstract class MultiLevelSkipList
// the skip interval in the list with level = 0
private int skipInterval;
+
+ // skipInterval used for level > 0
+ private int skipMultiplier;
// for every skip level a different buffer is used
private RAMOutputStream[] skipBuffer;
- protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+ protected MultiLevelSkipListWriter(int skipInterval, int skipMultiplier, int maxSkipLevels, int df) {
this.skipInterval = skipInterval;
+ this.skipMultiplier = skipMultiplier;
// calculate the maximum number of skip levels for this document frequency
- numberOfSkipLevels = MathUtil.log(df, skipInterval);
+ if (df <= skipInterval) {
+ numberOfSkipLevels = 1;
+ } else {
+ numberOfSkipLevels = 1+MathUtil.log(df/skipInterval, skipMultiplier);
+ }
// make sure it does not exceed maxSkipLevels
if (numberOfSkipLevels > maxSkipLevels) {
@@ -70,6 +81,11 @@ public abstract class MultiLevelSkipList
}
}
+ // skipMultiplier and skipInterval are the same:
+ protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+ this(skipInterval, skipInterval, maxSkipLevels, df);
+ }
+
protected void init() {
skipBuffer = new RAMOutputStream[numberOfSkipLevels];
for (int i = 0; i < numberOfSkipLevels; i++) {
@@ -95,7 +111,7 @@ public abstract class MultiLevelSkipList
* @param skipBuffer the skip buffer to write to
*/
protected abstract void writeSkipData(int level, IndexOutput skipBuffer) throws IOException;
-
+
/**
* Writes the current skip data to the buffers. The current document frequency determines
* the max level is skip data is to be written to.
@@ -104,11 +120,15 @@ public abstract class MultiLevelSkipList
* @throws IOException
*/
public void bufferSkip(int df) throws IOException {
- int numLevels;
+
+ assert df % skipInterval == 0;
+ int numLevels = 1;
+ df /= skipInterval;
// determine max level
- for (numLevels = 0; (df % skipInterval) == 0 && numLevels < numberOfSkipLevels; df /= skipInterval) {
+ while ((df % skipMultiplier) == 0 && numLevels < numberOfSkipLevels) {
numLevels++;
+ df /= skipMultiplier;
}
long childPointer = 0;
@@ -150,5 +170,4 @@ public abstract class MultiLevelSkipList
return skipPointer;
}
-
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/FixedIntBlockIndexInput.java Tue Aug 21 12:14:39 2012
@@ -77,76 +77,48 @@ public abstract class FixedIntBlockIndex
private static class Reader extends IntIndexInput.Reader {
private final IndexInput in;
+ private final BlockReader blockReader;
+ private final int blockSize;
+ private final int[] pending;
- protected final int[] pending;
- int upto;
-
+ private int upto;
private boolean seekPending;
private long pendingFP;
- private int pendingUpto;
- private long lastBlockFP;
- private final BlockReader blockReader;
- private final int blockSize;
- private final IntsRef bulkResult = new IntsRef();
+ private long lastBlockFP = -1;
public Reader(final IndexInput in, final int[] pending, final BlockReader blockReader) {
this.in = in;
this.pending = pending;
this.blockSize = pending.length;
- bulkResult.ints = pending;
this.blockReader = blockReader;
upto = blockSize;
}
void seek(final long fp, final int upto) {
- pendingFP = fp;
- pendingUpto = upto;
- seekPending = true;
- }
-
- private void maybeSeek() throws IOException {
- if (seekPending) {
- if (pendingFP != lastBlockFP) {
- // need new block
- in.seek(pendingFP);
- lastBlockFP = pendingFP;
- blockReader.readBlock();
- }
- upto = pendingUpto;
- seekPending = false;
+ assert upto < blockSize;
+ if (seekPending || fp != lastBlockFP) {
+ pendingFP = fp;
+ seekPending = true;
}
+ this.upto = upto;
}
@Override
public int next() throws IOException {
- this.maybeSeek();
- if (upto == blockSize) {
+ if (seekPending) {
+ // Seek & load new block
+ in.seek(pendingFP);
+ lastBlockFP = pendingFP;
+ blockReader.readBlock();
+ seekPending = false;
+ } else if (upto == blockSize) {
+ // Load new block
lastBlockFP = in.getFilePointer();
blockReader.readBlock();
upto = 0;
}
-
return pending[upto++];
}
-
- @Override
- public IntsRef read(final int count) throws IOException {
- this.maybeSeek();
- if (upto == blockSize) {
- blockReader.readBlock();
- upto = 0;
- }
- bulkResult.offset = upto;
- if (upto + count < blockSize) {
- bulkResult.length = count;
- upto += count;
- } else {
- bulkResult.length = blockSize - upto;
- upto = blockSize;
- }
-
- return bulkResult;
- }
}
private class Index extends IntIndexInput.Index {
@@ -178,7 +150,7 @@ public abstract class FixedIntBlockIndex
}
@Override
- public void set(final IntIndexInput.Index other) {
+ public void copyFrom(final IntIndexInput.Index other) {
final Index idx = (Index) other;
fp = idx.fp;
upto = idx.upto;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/intblock/VariableIntBlockIndexInput.java Tue Aug 21 12:14:39 2012
@@ -90,12 +90,10 @@ public abstract class VariableIntBlockIn
private long lastBlockFP;
private int blockSize;
private final BlockReader blockReader;
- private final IntsRef bulkResult = new IntsRef();
public Reader(final IndexInput in, final int[] pending, final BlockReader blockReader) {
this.in = in;
this.pending = pending;
- bulkResult.ints = pending;
this.blockReader = blockReader;
}
@@ -146,26 +144,6 @@ public abstract class VariableIntBlockIn
return pending[upto++];
}
-
- @Override
- public IntsRef read(final int count) throws IOException {
- this.maybeSeek();
- if (upto == blockSize) {
- lastBlockFP = in.getFilePointer();
- blockSize = blockReader.readBlock();
- upto = 0;
- }
- bulkResult.offset = upto;
- if (upto + count < blockSize) {
- bulkResult.length = count;
- upto += count;
- } else {
- bulkResult.length = blockSize - upto;
- upto = blockSize;
- }
-
- return bulkResult;
- }
}
private class Index extends IntIndexInput.Index {
@@ -204,7 +182,7 @@ public abstract class VariableIntBlockIn
}
@Override
- public void set(final IntIndexInput.Index other) {
+ public void copyFrom(final IntIndexInput.Index other) {
final Index idx = (Index) other;
fp = idx.fp;
upto = idx.upto;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java Tue Aug 21 12:14:39 2012
@@ -159,7 +159,7 @@ import org.apache.lucene.util.fst.FST; /
* with the frequency of the term in that document (except when frequencies are
* omitted: {@link IndexOptions#DOCS_ONLY}).</p>
* <ul>
- * <li>FreqFile (.frq) --> Header, <TermFreqs, SkipData> <sup>TermCount</sup></li>
+ * <li>FreqFile (.frq) --> Header, <TermFreqs, SkipData?> <sup>TermCount</sup></li>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>TermFreqs --> <TermFreq> <sup>DocFreq</sup></li>
* <li>TermFreq --> DocDelta[, Freq?]</li>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsWriter.java Tue Aug 21 12:14:39 2012
@@ -185,8 +185,6 @@ public final class Lucene40PostingsWrite
int lastDocID;
int df;
- /** Adds a new doc in this term. If this returns null
- * then we just skip consuming positions/payloads. */
@Override
public void startDoc(int docID, int termDocFreq) throws IOException {
// if (DEBUG) System.out.println("SPW: startDoc seg=" + segment + " docID=" + docID + " tf=" + termDocFreq + " freqOut.fp=" + freqOut.getFilePointer());
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/IntIndexInput.java Tue Aug 21 12:14:39 2012
@@ -44,7 +44,7 @@ public abstract class IntIndexInput impl
/** Seeks primary stream to the last read offset */
public abstract void seek(IntIndexInput.Reader stream) throws IOException;
- public abstract void set(Index other);
+ public abstract void copyFrom(Index other);
@Override
public abstract Index clone();
@@ -55,23 +55,5 @@ public abstract class IntIndexInput impl
/** Reads next single int */
public abstract int next() throws IOException;
-
- /** Reads next chunk of ints */
- private IntsRef bulkResult;
-
- /** Read up to count ints. */
- public IntsRef read(int count) throws IOException {
- if (bulkResult == null) {
- bulkResult = new IntsRef();
- bulkResult.ints = new int[count];
- } else {
- bulkResult.grow(count);
- }
- for(int i=0;i<count;i++) {
- bulkResult.ints[i] = next();
- }
- bulkResult.length = count;
- return bulkResult;
- }
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Tue Aug 21 12:14:39 2012
@@ -160,13 +160,13 @@ public class SepPostingsReader extends P
if (docIndex == null) {
docIndex = other.docIndex.clone();
} else {
- docIndex.set(other.docIndex);
+ docIndex.copyFrom(other.docIndex);
}
if (other.freqIndex != null) {
if (freqIndex == null) {
freqIndex = other.freqIndex.clone();
} else {
- freqIndex.set(other.freqIndex);
+ freqIndex.copyFrom(other.freqIndex);
}
} else {
freqIndex = null;
@@ -175,7 +175,7 @@ public class SepPostingsReader extends P
if (posIndex == null) {
posIndex = other.posIndex.clone();
} else {
- posIndex.set(other.posIndex);
+ posIndex.copyFrom(other.posIndex);
}
} else {
posIndex = null;
@@ -352,11 +352,11 @@ public class SepPostingsReader extends P
// TODO: can't we only do this if consumer
// skipped consuming the previous docs?
- docIndex.set(termState.docIndex);
+ docIndex.copyFrom(termState.docIndex);
docIndex.seek(docReader);
if (!omitTF) {
- freqIndex.set(termState.freqIndex);
+ freqIndex.copyFrom(termState.freqIndex);
freqIndex.seek(freqReader);
}
@@ -516,15 +516,15 @@ public class SepPostingsReader extends P
// TODO: can't we only do this if consumer
// skipped consuming the previous docs?
- docIndex.set(termState.docIndex);
+ docIndex.copyFrom(termState.docIndex);
docIndex.seek(docReader);
//System.out.println(" docIndex=" + docIndex);
- freqIndex.set(termState.freqIndex);
+ freqIndex.copyFrom(termState.freqIndex);
freqIndex.seek(freqReader);
//System.out.println(" freqIndex=" + freqIndex);
- posIndex.set(termState.posIndex);
+ posIndex.copyFrom(termState.posIndex);
//System.out.println(" posIndex=" + posIndex);
posSeekPending = true;
payloadPending = false;
@@ -629,7 +629,7 @@ public class SepPostingsReader extends P
// NOTE: don't seek pos here; do it lazily
// instead. Eg a PhraseQuery may skip to many
// docs before finally asking for positions...
- posIndex.set(skipper.getPosIndex());
+ posIndex.copyFrom(skipper.getPosIndex());
posSeekPending = true;
count = newCount;
doc = accum = skipper.getDoc();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepSkipListReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepSkipListReader.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepSkipListReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/sep/SepSkipListReader.java Tue Aug 21 12:14:39 2012
@@ -108,12 +108,12 @@ class SepSkipListReader extends MultiLev
lastPayloadPointer = payloadBasePointer;
for(int i=0;i<maxNumberOfSkipLevels;i++) {
- docIndex[i].set(docBaseIndex);
+ docIndex[i].copyFrom(docBaseIndex);
if (freqIndex != null) {
- freqIndex[i].set(freqBaseIndex);
+ freqIndex[i].copyFrom(freqBaseIndex);
}
if (posBaseIndex != null) {
- posIndex[i].set(posBaseIndex);
+ posIndex[i].copyFrom(posBaseIndex);
}
}
Arrays.fill(payloadPointer, payloadBasePointer);
@@ -145,20 +145,20 @@ class SepSkipListReader extends MultiLev
lastPayloadPointer = payloadPointer[level];
lastPayloadLength = payloadLength[level];
if (freqIndex != null) {
- lastFreqIndex.set(freqIndex[level]);
+ lastFreqIndex.copyFrom(freqIndex[level]);
}
- lastDocIndex.set(docIndex[level]);
+ lastDocIndex.copyFrom(docIndex[level]);
if (lastPosIndex != null) {
- lastPosIndex.set(posIndex[level]);
+ lastPosIndex.copyFrom(posIndex[level]);
}
if (level > 0) {
if (freqIndex != null) {
- freqIndex[level-1].set(freqIndex[level]);
+ freqIndex[level-1].copyFrom(freqIndex[level]);
}
- docIndex[level-1].set(docIndex[level]);
+ docIndex[level-1].copyFrom(docIndex[level]);
if (posIndex != null) {
- posIndex[level-1].set(posIndex[level]);
+ posIndex[level-1].copyFrom(posIndex[level]);
}
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java?rev=1375495&r1=1375494&r2=1375495&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FieldInfos.java Tue Aug 21 12:14:39 2012
@@ -34,6 +34,8 @@ import org.apache.lucene.index.FieldInfo
public class FieldInfos implements Iterable<FieldInfo> {
private final boolean hasFreq;
private final boolean hasProx;
+ private final boolean hasPayloads;
+ private final boolean hasOffsets;
private final boolean hasVectors;
private final boolean hasNorms;
private final boolean hasDocValues;
@@ -45,6 +47,8 @@ public class FieldInfos implements Itera
public FieldInfos(FieldInfo[] infos) {
boolean hasVectors = false;
boolean hasProx = false;
+ boolean hasPayloads = false;
+ boolean hasOffsets = false;
boolean hasFreq = false;
boolean hasNorms = false;
boolean hasDocValues = false;
@@ -58,12 +62,16 @@ public class FieldInfos implements Itera
hasVectors |= info.hasVectors();
hasProx |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
hasFreq |= info.isIndexed() && info.getIndexOptions() != IndexOptions.DOCS_ONLY;
+ hasOffsets |= info.isIndexed() && info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
hasNorms |= info.hasNorms();
hasDocValues |= info.hasDocValues();
+ hasPayloads |= info.hasPayloads();
}
this.hasVectors = hasVectors;
this.hasProx = hasProx;
+ this.hasPayloads = hasPayloads;
+ this.hasOffsets = hasOffsets;
this.hasFreq = hasFreq;
this.hasNorms = hasNorms;
this.hasDocValues = hasDocValues;
@@ -79,6 +87,16 @@ public class FieldInfos implements Itera
public boolean hasProx() {
return hasProx;
}
+
+ /** Returns true if any fields have payloads */
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
+
+ /** Returns true if any fields have offsets */
+ public boolean hasOffsets() {
+ return hasOffsets;
+ }
/**
* @return true if at least one field has any vectors