You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/08/11 21:37:45 UTC
svn commit: r1372010 [1/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/codecs/
lucene/core/src/java/org/apache/lucene/codecs/bloom/
lucene/core/src/java/org/apache/lucene/codecs/lucene3x/ lucene/c...
Author: rmuir
Date: Sat Aug 11 19:37:44 2012
New Revision: 1372010
URL: http://svn.apache.org/viewvc?rev=1372010&view=rev
Log:
LUCENE-1888: added option to store payloads on the term vectors
Added:
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
- copied, changed from r1371996, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/Terms.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
lucene/dev/branches/branch_4x/lucene/memory/ (props changed)
lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/branch_4x/lucene/test-framework/ (props changed)
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java
lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Sat Aug 11 19:37:44 2012
@@ -8,6 +8,13 @@ http://svn.apache.org/repos/asf/lucene/d
======================= Lucene 4.0.0 =======================
+New Features
+
+* LUCENE-1888: Added the option to store payloads in the term
+ vectors (IndexableFieldType.storeTermVectorPayloads()). Note
+ that you must store term vector positions to store payloads.
+ (Robert Muir)
+
API Changes
* LUCENE-4299: Added Terms.hasPositions() and Terms.hasOffsets().
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java Sat Aug 11 19:37:44 2012
@@ -262,6 +262,11 @@ public class BlockTermsReader extends Fi
public boolean hasPositions() {
return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return fieldInfo.hasPayloads();
+ }
@Override
public long size() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java Sat Aug 11 19:37:44 2012
@@ -465,6 +465,11 @@ public class BlockTreeTermsReader extend
public boolean hasPositions() {
return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return fieldInfo.hasPayloads();
+ }
@Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/MappingMultiDocsAndPositionsEnum.java Sat Aug 11 19:37:44 2012
@@ -124,14 +124,22 @@ public final class MappingMultiDocsAndPo
@Override
public BytesRef getPayload() throws IOException {
BytesRef payload = current.getPayload();
- if (mergeState.currentPayloadProcessor[upto] != null) {
+ if (mergeState.currentPayloadProcessor[upto] != null && payload != null) {
+ // to not violate the D&P api, we must give the processor a private copy
+ payload = BytesRef.deepCopyOf(payload);
mergeState.currentPayloadProcessor[upto].processPayload(payload);
+ if (payload.length == 0) {
+ // don't let PayloadProcessors corrumpt the index
+ return null;
+ }
}
return payload;
}
@Override
public boolean hasPayload() {
+ // TODO: note this is actually bogus if there is a payloadProcessor,
+ // because it might remove it: but lets just remove this method completely
return current.hasPayload();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java Sat Aug 11 19:37:44 2012
@@ -28,6 +28,8 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.MergeState;
+import org.apache.lucene.index.PayloadProcessorProvider.PayloadProcessor;
+import org.apache.lucene.index.PayloadProcessorProvider.ReaderPayloadProcessor;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
@@ -41,14 +43,14 @@ import org.apache.lucene.util.BytesRef;
* <ol>
* <li>For every document, {@link #startDocument(int)} is called,
* informing the Codec how many fields will be written.
- * <li>{@link #startField(FieldInfo, int, boolean, boolean)} is called for
+ * <li>{@link #startField(FieldInfo, int, boolean, boolean, boolean)} is called for
* each field in the document, informing the codec how many terms
- * will be written for that field, and whether or not positions
- * or offsets are enabled.
+ * will be written for that field, and whether or not positions,
+ * offsets, or payloads are enabled.
* <li>Within each field, {@link #startTerm(BytesRef, int)} is called
* for each term.
* <li>If offsets and/or positions are enabled, then
- * {@link #addPosition(int, int, int)} will be called for each term
+ * {@link #addPosition(int, int, int, BytesRef)} will be called for each term
* occurrence.
* <li>After all documents have been written, {@link #finish(FieldInfos, int)}
* is called for verification/sanity-checks.
@@ -60,7 +62,7 @@ import org.apache.lucene.util.BytesRef;
public abstract class TermVectorsWriter implements Closeable {
/** Called before writing the term vectors of the document.
- * {@link #startField(FieldInfo, int, boolean, boolean)} will
+ * {@link #startField(FieldInfo, int, boolean, boolean, boolean)} will
* be called <code>numVectorFields</code> times. Note that if term
* vectors are enabled, this is called even if the document
* has no vector fields, in this case <code>numVectorFields</code>
@@ -69,17 +71,17 @@ public abstract class TermVectorsWriter
/** Called before writing the terms of the field.
* {@link #startTerm(BytesRef, int)} will be called <code>numTerms</code> times. */
- public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException;
+ public abstract void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException;
/** Adds a term and its term frequency <code>freq</code>.
* If this field has positions and/or offsets enabled, then
- * {@link #addPosition(int, int, int)} will be called
+ * {@link #addPosition(int, int, int, BytesRef)} will be called
* <code>freq</code> times respectively.
*/
public abstract void startTerm(BytesRef term, int freq) throws IOException;
/** Adds a term position and offsets */
- public abstract void addPosition(int position, int startOffset, int endOffset) throws IOException;
+ public abstract void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException;
/** Aborts writing entirely, implementation should remove
* any partially-written files, etc. */
@@ -99,7 +101,7 @@ public abstract class TermVectorsWriter
* This is an expert API that allows the codec to consume
* positions and offsets directly from the indexer.
* <p>
- * The default implementation calls {@link #addPosition(int, int, int)},
+ * The default implementation calls {@link #addPosition(int, int, int, BytesRef)},
* but subclasses can override this if they want to efficiently write
* all the positions, then all the offsets, for example.
* <p>
@@ -111,15 +113,36 @@ public abstract class TermVectorsWriter
public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
int position = 0;
int lastOffset = 0;
+ BytesRef payload = null;
for (int i = 0; i < numProx; i++) {
final int startOffset;
final int endOffset;
+ final BytesRef thisPayload;
if (positions == null) {
position = -1;
+ thisPayload = null;
} else {
- position += positions.readVInt();
+ int code = positions.readVInt();
+ position += code >>> 1;
+ if ((code & 1) != 0) {
+ // This position has a payload
+ final int payloadLength = positions.readVInt();
+
+ if (payload == null) {
+ payload = new BytesRef();
+ payload.bytes = new byte[payloadLength];
+ } else if (payload.bytes.length < payloadLength) {
+ payload.grow(payloadLength);
+ }
+
+ positions.readBytes(payload.bytes, 0, payloadLength);
+ payload.length = payloadLength;
+ thisPayload = payload;
+ } else {
+ thisPayload = null;
+ }
}
if (offsets == null) {
@@ -129,24 +152,31 @@ public abstract class TermVectorsWriter
endOffset = startOffset + offsets.readVInt();
lastOffset = endOffset;
}
- addPosition(position, startOffset, endOffset);
+ addPosition(position, startOffset, endOffset, thisPayload);
}
}
/** Merges in the term vectors from the readers in
* <code>mergeState</code>. The default implementation skips
* over deleted documents, and uses {@link #startDocument(int)},
- * {@link #startField(FieldInfo, int, boolean, boolean)},
- * {@link #startTerm(BytesRef, int)}, {@link #addPosition(int, int, int)},
+ * {@link #startField(FieldInfo, int, boolean, boolean, boolean)},
+ * {@link #startTerm(BytesRef, int)}, {@link #addPosition(int, int, int, BytesRef)},
* and {@link #finish(FieldInfos, int)},
* returning the number of documents that were written.
* Implementations can override this method for more sophisticated
* merging (bulk-byte copying, etc). */
public int merge(MergeState mergeState) throws IOException {
int docCount = 0;
- for (AtomicReader reader : mergeState.readers) {
+ for (int i = 0; i < mergeState.readers.size(); i++) {
+ final AtomicReader reader = mergeState.readers.get(i);
final int maxDoc = reader.maxDoc();
final Bits liveDocs = reader.getLiveDocs();
+ // set PayloadProcessor
+ if (mergeState.payloadProcessorProvider != null) {
+ mergeState.currentReaderPayloadProcessor = mergeState.readerPayloadProcessor[i];
+ } else {
+ mergeState.currentReaderPayloadProcessor = null;
+ }
for (int docID = 0; docID < maxDoc; docID++) {
if (liveDocs != null && !liveDocs.get(docID)) {
// skip deleted docs
@@ -155,7 +185,7 @@ public abstract class TermVectorsWriter
// NOTE: it's very important to first assign to vectors then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
Fields vectors = reader.getTermVectors(docID);
- addAllDocVectors(vectors, mergeState.fieldInfos);
+ addAllDocVectors(vectors, mergeState);
docCount++;
mergeState.checkAbort.work(300);
}
@@ -169,7 +199,7 @@ public abstract class TermVectorsWriter
* implementation requires that the vectors implement
* both Fields.size and
* Terms.size. */
- protected final void addAllDocVectors(Fields vectors, FieldInfos fieldInfos) throws IOException {
+ protected final void addAllDocVectors(Fields vectors, MergeState mergeState) throws IOException {
if (vectors == null) {
startDocument(0);
return;
@@ -187,9 +217,12 @@ public abstract class TermVectorsWriter
TermsEnum termsEnum = null;
DocsAndPositionsEnum docsAndPositionsEnum = null;
+
+ final ReaderPayloadProcessor readerPayloadProcessor = mergeState.currentReaderPayloadProcessor;
+ PayloadProcessor payloadProcessor = null;
while((fieldName = fieldsEnum.next()) != null) {
- final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldName);
+ final FieldInfo fieldInfo = mergeState.fieldInfos.fieldInfo(fieldName);
assert lastFieldName == null || fieldName.compareTo(lastFieldName) > 0: "lastFieldName=" + lastFieldName + " fieldName=" + fieldName;
lastFieldName = fieldName;
@@ -202,13 +235,15 @@ public abstract class TermVectorsWriter
final boolean hasPositions = terms.hasPositions();
final boolean hasOffsets = terms.hasOffsets();
+ final boolean hasPayloads = terms.hasPayloads();
+ assert !hasPayloads || hasPositions;
final int numTerms = (int) terms.size();
if (numTerms == -1) {
throw new IllegalStateException("terms.size() must be implemented (it returned -1)");
}
- startField(fieldInfo, numTerms, hasPositions, hasOffsets);
+ startField(fieldInfo, numTerms, hasPositions, hasOffsets, hasPayloads);
termsEnum = terms.iterator(termsEnum);
int termCount = 0;
@@ -218,6 +253,10 @@ public abstract class TermVectorsWriter
final int freq = (int) termsEnum.totalTermFreq();
startTerm(termsEnum.term(), freq);
+
+ if (hasPayloads && readerPayloadProcessor != null) {
+ payloadProcessor = readerPayloadProcessor.getProcessor(fieldName, termsEnum.term());
+ }
if (hasPositions || hasOffsets) {
docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum);
@@ -231,9 +270,22 @@ public abstract class TermVectorsWriter
final int pos = docsAndPositionsEnum.nextPosition();
final int startOffset = docsAndPositionsEnum.startOffset();
final int endOffset = docsAndPositionsEnum.endOffset();
+
+ BytesRef payload = docsAndPositionsEnum.hasPayload() ?
+ docsAndPositionsEnum.getPayload() : null;
+
+ if (payloadProcessor != null && payload != null) {
+ // to not violate the D&P api, we must give the processor a private copy
+ payload = BytesRef.deepCopyOf(payload);
+ payloadProcessor.processPayload(payload);
+ if (payload.length == 0) {
+ // don't let PayloadProcessors corrumpt the index
+ payload = null;
+ }
+ }
assert !hasPositions || pos >= 0;
- addPosition(pos, startOffset, endOffset);
+ addPosition(pos, startOffset, endOffset, payload);
}
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Sat Aug 11 19:37:44 2012
@@ -324,6 +324,11 @@ public class BloomFilteringPostingsForma
public boolean hasPositions() {
return delegateTerms.hasPositions();
}
+
+ @Override
+ public boolean hasPayloads() {
+ return delegateTerms.hasPayloads();
+ }
}
class BloomFilteredTermsEnum extends TermsEnum {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xFields.java Sat Aug 11 19:37:44 2012
@@ -258,6 +258,11 @@ class Lucene3xFields extends FieldsProdu
public boolean hasPositions() {
return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return fieldInfo.hasPayloads();
+ }
}
private class PreTermsEnum extends TermsEnum {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene3x/Lucene3xTermVectorsReader.java Sat Aug 11 19:37:44 2012
@@ -349,6 +349,11 @@ class Lucene3xTermVectorsReader extends
public boolean hasPositions() {
return storePositions;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
}
static class TermAndPostings {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java Sat Aug 11 19:37:44 2012
@@ -67,33 +67,46 @@ import org.apache.lucene.store.IOContext
* <li><a name="tvf" id="tvf"></a>
* <p>The Field or .tvf file.</p>
* <p>This file contains, for each field that has a term vector stored, a list of
- * the terms, their frequencies and, optionally, position and offset
+ * the terms, their frequencies and, optionally, position, offset, and payload
* information.</p>
- * <p>Field (.tvf) --> Header,<NumTerms, Position/Offset, TermFreqs>
+ * <p>Field (.tvf) --> Header,<NumTerms, Flags, TermFreqs>
* <sup>NumFields</sup></p>
* <ul>
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
* <li>NumTerms --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Position/Offset --> {@link DataOutput#writeByte Byte}</li>
- * <li>TermFreqs --> <TermText, TermFreq, Positions?, Offsets?>
+ * <li>Flags --> {@link DataOutput#writeByte Byte}</li>
+ * <li>TermFreqs --> <TermText, TermFreq, Positions?, PayloadData?, Offsets?>
* <sup>NumTerms</sup></li>
* <li>TermText --> <PrefixLength, Suffix></li>
* <li>PrefixLength --> {@link DataOutput#writeVInt VInt}</li>
* <li>Suffix --> {@link DataOutput#writeString String}</li>
* <li>TermFreq --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Positions --> <{@link DataOutput#writeVInt VInt}><sup>TermFreq</sup></li>
+ * <li>Positions --> <PositionDelta PayloadLength?><sup>TermFreq</sup></li>
+ * <li>PositionDelta --> {@link DataOutput#writeVInt VInt}</li>
+ * <li>PayloadLength --> {@link DataOutput#writeVInt VInt}</li>
+ * <li>PayloadData --> {@link DataOutput#writeByte Byte}<sup>NumPayloadBytes</sup></li>
* <li>Offsets --> <{@link DataOutput#writeVInt VInt}, {@link DataOutput#writeVInt VInt}><sup>TermFreq</sup></li>
* </ul>
* <p>Notes:</p>
* <ul>
- * <li>Position/Offset byte stores whether this term vector has position or offset
+ * <li>Flags byte stores whether this term vector has position, offset, payload.
* information stored.</li>
* <li>Term byte prefixes are shared. The PrefixLength is the number of initial
* bytes from the previous term which must be pre-pended to a term's suffix
* in order to form the term's bytes. Thus, if the previous term's text was "bone"
* and the term is "boy", the PrefixLength is two and the suffix is "y".</li>
- * <li>Positions are stored as delta encoded VInts. This means we only store the
- * difference of the current position from the last position</li>
+ * <li>PositionDelta is, if payloads are disabled for the term's field, the
+ * difference between the position of the current occurrence in the document and
+ * the previous occurrence (or zero, if this is the first occurrence in this
+ * document). If payloads are enabled for the term's field, then PositionDelta/2
+ * is the difference between the current and the previous position. If payloads
+ * are enabled and PositionDelta is odd, then PayloadLength is stored, indicating
+ * the length of the payload at the current term position.</li>
+ * <li>PayloadData is metadata associated with a term position. If
+ * PayloadLength is stored at the current position, then it indicates the length
+ * of this payload. If PayloadLength is not stored, then this payload has the same
+ * length as the payload at the previous position. PayloadData encodes the
+ * concatenated bytes for all of a terms occurrences.</li>
* <li>Offsets are stored as delta encoded VInts. The first VInt is the
* startOffset, the second is the endOffset.</li>
* </ul>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java Sat Aug 11 19:37:44 2012
@@ -55,6 +55,8 @@ public class Lucene40TermVectorsReader e
static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x2;
+ static final byte STORE_PAYLOAD_WITH_TERMVECTOR = 0x4;
+
/** Extension of vectors fields file */
static final String VECTORS_FIELDS_EXTENSION = "tvf";
@@ -68,8 +70,10 @@ public class Lucene40TermVectorsReader e
static final String CODEC_NAME_DOCS = "Lucene40TermVectorsDocs";
static final String CODEC_NAME_INDEX = "Lucene40TermVectorsIndex";
- static final int VERSION_START = 0;
- static final int VERSION_CURRENT = VERSION_START;
+ static final int VERSION_NO_PAYLOADS = 0;
+ static final int VERSION_PAYLOADS = 1;
+ static final int VERSION_START = VERSION_NO_PAYLOADS;
+ static final int VERSION_CURRENT = VERSION_PAYLOADS;
static final long HEADER_LENGTH_FIELDS = CodecUtil.headerLength(CODEC_NAME_FIELDS);
static final long HEADER_LENGTH_DOCS = CodecUtil.headerLength(CODEC_NAME_DOCS);
@@ -298,7 +302,7 @@ public class Lucene40TermVectorsReader e
private final long tvfFPStart;
private final boolean storePositions;
private final boolean storeOffsets;
-
+ private final boolean storePayloads;
public TVTerms(long tvfFP) throws IOException {
tvf.seek(tvfFP);
@@ -306,6 +310,7 @@ public class Lucene40TermVectorsReader e
final byte bits = tvf.readByte();
storePositions = (bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0;
storeOffsets = (bits & STORE_OFFSET_WITH_TERMVECTOR) != 0;
+ storePayloads = (bits & STORE_PAYLOAD_WITH_TERMVECTOR) != 0;
tvfFPStart = tvf.getFilePointer();
}
@@ -320,7 +325,7 @@ public class Lucene40TermVectorsReader e
} else {
termsEnum = new TVTermsEnum();
}
- termsEnum.reset(numTerms, tvfFPStart, storePositions, storeOffsets);
+ termsEnum.reset(numTerms, tvfFPStart, storePositions, storeOffsets, storePayloads);
return termsEnum;
}
@@ -361,6 +366,11 @@ public class Lucene40TermVectorsReader e
public boolean hasPositions() {
return storePositions;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return storePayloads;
+ }
}
private class TVTermsEnum extends TermsEnum {
@@ -373,11 +383,17 @@ public class Lucene40TermVectorsReader e
private BytesRef term = new BytesRef();
private boolean storePositions;
private boolean storeOffsets;
+ private boolean storePayloads;
private long tvfFP;
private int[] positions;
private int[] startOffsets;
private int[] endOffsets;
+
+ // one shared byte[] for any term's payloads
+ private int[] payloadOffsets;
+ private int lastPayloadLength;
+ private byte[] payloadData;
// NOTE: tvf is pre-positioned by caller
public TVTermsEnum() {
@@ -389,16 +405,20 @@ public class Lucene40TermVectorsReader e
return tvf == origTVF;
}
- public void reset(int numTerms, long tvfFPStart, boolean storePositions, boolean storeOffsets) throws IOException {
+ public void reset(int numTerms, long tvfFPStart, boolean storePositions, boolean storeOffsets, boolean storePayloads) throws IOException {
this.numTerms = numTerms;
this.storePositions = storePositions;
this.storeOffsets = storeOffsets;
+ this.storePayloads = storePayloads;
nextTerm = 0;
tvf.seek(tvfFPStart);
tvfFP = 1+tvfFPStart;
positions = null;
startOffsets = null;
endOffsets = null;
+ payloadOffsets = null;
+ payloadData = null;
+ lastPayloadLength = -1;
}
// NOTE: slow! (linear scan)
@@ -445,7 +465,26 @@ public class Lucene40TermVectorsReader e
tvf.readBytes(term.bytes, start, deltaLen);
freq = tvf.readVInt();
- if (storePositions) {
+ if (storePayloads) {
+ positions = new int[freq];
+ payloadOffsets = new int[freq];
+ int totalPayloadLength = 0;
+ int pos = 0;
+ for(int posUpto=0;posUpto<freq;posUpto++) {
+ int code = tvf.readVInt();
+ pos += code >>> 1;
+ positions[posUpto] = pos;
+ if ((code & 1) != 0) {
+ // length change
+ lastPayloadLength = tvf.readVInt();
+ }
+ payloadOffsets[posUpto] = totalPayloadLength;
+ totalPayloadLength += lastPayloadLength;
+ assert totalPayloadLength >= 0;
+ }
+ payloadData = new byte[totalPayloadLength];
+ tvf.readBytes(payloadData, 0, payloadData.length);
+ } else if (storePositions /* no payloads */) {
// TODO: we could maybe reuse last array, if we can
// somehow be careful about consumer never using two
// D&PEnums at once...
@@ -517,14 +556,12 @@ public class Lucene40TermVectorsReader e
} else {
docsAndPositionsEnum = new TVDocsAndPositionsEnum();
}
- docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets);
+ docsAndPositionsEnum.reset(liveDocs, positions, startOffsets, endOffsets, payloadOffsets, payloadData);
return docsAndPositionsEnum;
}
@Override
public Comparator<BytesRef> getComparator() {
- // TODO: really indexer hardwires
- // this...? I guess codec could buffer and re-sort...
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
}
@@ -582,6 +619,9 @@ public class Lucene40TermVectorsReader e
private int[] positions;
private int[] startOffsets;
private int[] endOffsets;
+ private int[] payloadOffsets;
+ private BytesRef payload = new BytesRef();
+ private byte[] payloadBytes;
@Override
public int freq() throws IOException {
@@ -617,11 +657,13 @@ public class Lucene40TermVectorsReader e
}
}
- public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
+ public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets, int[] payloadLengths, byte[] payloadBytes) {
this.liveDocs = liveDocs;
this.positions = positions;
this.startOffsets = startOffsets;
this.endOffsets = endOffsets;
+ this.payloadOffsets = payloadLengths;
+ this.payloadBytes = payloadBytes;
this.doc = -1;
didNext = false;
nextPos = 0;
@@ -629,12 +671,31 @@ public class Lucene40TermVectorsReader e
@Override
public BytesRef getPayload() {
- return null;
+ // TODO: dumb that we have to duplicate hasPayload
+ if (payloadOffsets == null) {
+ return null;
+ } else {
+ int off = payloadOffsets[nextPos-1];
+ int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
+ if (end - off == 0) {
+ return null;
+ }
+ payload.bytes = payloadBytes;
+ payload.offset = off;
+ payload.length = end - off;
+ return payload;
+ }
}
@Override
public boolean hasPayload() {
- return false;
+ if (payloadOffsets == null) {
+ return false;
+ } else {
+ int off = payloadOffsets[nextPos-1];
+ int end = nextPos == payloadOffsets.length ? payloadBytes.length : payloadOffsets[nextPos];
+ return end - off > 0;
+ }
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java Sat Aug 11 19:37:44 2012
@@ -106,12 +106,14 @@ public final class Lucene40TermVectorsWr
private String lastFieldName;
@Override
- public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
+ public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
assert lastFieldName == null || info.name.compareTo(lastFieldName) > 0: "fieldName=" + info.name + " lastFieldName=" + lastFieldName;
lastFieldName = info.name;
this.positions = positions;
this.offsets = offsets;
+ this.payloads = payloads;
lastTerm.length = 0;
+ lastPayloadLength = -1; // force first payload to write its length
fps[fieldCount++] = tvf.getFilePointer();
tvd.writeVInt(info.number);
tvf.writeVInt(numTerms);
@@ -120,6 +122,8 @@ public final class Lucene40TermVectorsWr
bits |= Lucene40TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
if (offsets)
bits |= Lucene40TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
+ if (payloads)
+ bits |= Lucene40TermVectorsReader.STORE_PAYLOAD_WITH_TERMVECTOR;
tvf.writeByte(bits);
assert fieldCount <= numVectorFields;
@@ -138,10 +142,12 @@ public final class Lucene40TermVectorsWr
// we also don't buffer during bulk merges.
private int offsetStartBuffer[] = new int[10];
private int offsetEndBuffer[] = new int[10];
- private int offsetIndex = 0;
- private int offsetFreq = 0;
+ private BytesRef payloadData = new BytesRef(10);
+ private int bufferedIndex = 0;
+ private int bufferedFreq = 0;
private boolean positions = false;
private boolean offsets = false;
+ private boolean payloads = false;
@Override
public void startTerm(BytesRef term, int freq) throws IOException {
@@ -158,20 +164,40 @@ public final class Lucene40TermVectorsWr
// we might need to buffer if its a non-bulk merge
offsetStartBuffer = ArrayUtil.grow(offsetStartBuffer, freq);
offsetEndBuffer = ArrayUtil.grow(offsetEndBuffer, freq);
- offsetIndex = 0;
- offsetFreq = freq;
}
+ bufferedIndex = 0;
+ bufferedFreq = freq;
+ payloadData.length = 0;
}
int lastPosition = 0;
int lastOffset = 0;
+ int lastPayloadLength = -1; // force first payload to write its length
+
+ BytesRef scratch = new BytesRef(); // used only by this optimized flush below
@Override
public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
- // TODO: technically we could just copy bytes and not re-encode if we knew the length...
- if (positions != null) {
+ if (payloads) {
+ // TODO, maybe overkill and just call super.addProx() in this case?
+ // we do avoid buffering the offsets in RAM though.
for (int i = 0; i < numProx; i++) {
- tvf.writeVInt(positions.readVInt());
+ int code = positions.readVInt();
+ if ((code & 1) == 1) {
+ int length = positions.readVInt();
+ scratch.grow(length);
+ scratch.length = length;
+ positions.readBytes(scratch.bytes, scratch.offset, scratch.length);
+ writePosition(code >>> 1, scratch);
+ } else {
+ writePosition(code >>> 1, null);
+ }
+ }
+ tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
+ } else if (positions != null) {
+ // pure positions, no payloads
+ for (int i = 0; i < numProx; i++) {
+ tvf.writeVInt(positions.readVInt() >>> 1);
}
}
@@ -184,28 +210,36 @@ public final class Lucene40TermVectorsWr
}
@Override
- public void addPosition(int position, int startOffset, int endOffset) throws IOException {
- if (positions && offsets) {
+ public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
+ if (positions && (offsets || payloads)) {
// write position delta
- tvf.writeVInt(position - lastPosition);
+ writePosition(position - lastPosition, payload);
lastPosition = position;
// buffer offsets
- offsetStartBuffer[offsetIndex] = startOffset;
- offsetEndBuffer[offsetIndex] = endOffset;
- offsetIndex++;
+ if (offsets) {
+ offsetStartBuffer[bufferedIndex] = startOffset;
+ offsetEndBuffer[bufferedIndex] = endOffset;
+ }
+
+ bufferedIndex++;
// dump buffer if we are done
- if (offsetIndex == offsetFreq) {
- for (int i = 0; i < offsetIndex; i++) {
- tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
- tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
- lastOffset = offsetEndBuffer[i];
+ if (bufferedIndex == bufferedFreq) {
+ if (payloads) {
+ tvf.writeBytes(payloadData.bytes, payloadData.offset, payloadData.length);
+ }
+ for (int i = 0; i < bufferedIndex; i++) {
+ if (offsets) {
+ tvf.writeVInt(offsetStartBuffer[i] - lastOffset);
+ tvf.writeVInt(offsetEndBuffer[i] - offsetStartBuffer[i]);
+ lastOffset = offsetEndBuffer[i];
+ }
}
}
} else if (positions) {
// write position delta
- tvf.writeVInt(position - lastPosition);
+ writePosition(position - lastPosition, payload);
lastPosition = position;
} else if (offsets) {
// write offset deltas
@@ -214,6 +248,30 @@ public final class Lucene40TermVectorsWr
lastOffset = endOffset;
}
}
+
+ private void writePosition(int delta, BytesRef payload) throws IOException {
+ if (payloads) {
+ int payloadLength = payload == null ? 0 : payload.length;
+
+ if (payloadLength != lastPayloadLength) {
+ lastPayloadLength = payloadLength;
+ tvf.writeVInt((delta<<1)|1);
+ tvf.writeVInt(payloadLength);
+ } else {
+ tvf.writeVInt(delta << 1);
+ }
+ if (payloadLength > 0) {
+ if (payloadLength + payloadData.length < 0) {
+ // we overflowed the payload buffer, just throw UOE
+ // having > Integer.MAX_VALUE bytes of payload for a single term in a single doc is nuts.
+ throw new UnsupportedOperationException("A term cannot have more than Integer.MAX_VALUE bytes of payload data in a single document");
+ }
+ payloadData.append(payload);
+ }
+ } else {
+ tvf.writeVInt(delta);
+ }
+ }
@Override
public void abort() {
@@ -255,7 +313,14 @@ public final class Lucene40TermVectorsWr
int idx = 0;
int numDocs = 0;
- for (final AtomicReader reader : mergeState.readers) {
+ for (int i = 0; i < mergeState.readers.size(); i++) {
+ final AtomicReader reader = mergeState.readers.get(i);
+ // set PayloadProcessor
+ if (mergeState.payloadProcessorProvider != null) {
+ mergeState.currentReaderPayloadProcessor = mergeState.readerPayloadProcessor[i];
+ } else {
+ mergeState.currentReaderPayloadProcessor = null;
+ }
final SegmentReader matchingSegmentReader = mergeState.matchingSegmentReaders[idx++];
Lucene40TermVectorsReader matchingVectorsReader = null;
if (matchingSegmentReader != null) {
@@ -288,8 +353,8 @@ public final class Lucene40TermVectorsWr
final int maxDoc = reader.maxDoc();
final Bits liveDocs = reader.getLiveDocs();
int totalNumDocs = 0;
- if (matchingVectorsReader != null) {
- // We can bulk-copy because the fieldInfos are "congruent"
+ if (matchingVectorsReader != null && mergeState.currentReaderPayloadProcessor == null) {
+ // We can bulk-copy because the fieldInfos are "congruent" and there is no payload processor
for (int docNum = 0; docNum < maxDoc;) {
if (!liveDocs.get(docNum)) {
// skip deleted docs
@@ -324,7 +389,7 @@ public final class Lucene40TermVectorsWr
// NOTE: it's very important to first assign to vectors then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
Fields vectors = reader.getTermVectors(docNum);
- addAllDocVectors(vectors, mergeState.fieldInfos);
+ addAllDocVectors(vectors, mergeState);
totalNumDocs++;
mergeState.checkAbort.work(300);
}
@@ -339,8 +404,8 @@ public final class Lucene40TermVectorsWr
int rawDocLengths2[])
throws IOException {
final int maxDoc = reader.maxDoc();
- if (matchingVectorsReader != null) {
- // We can bulk-copy because the fieldInfos are "congruent"
+ if (matchingVectorsReader != null && mergeState.currentReaderPayloadProcessor == null) {
+ // We can bulk-copy because the fieldInfos are "congruent" and there is no payload processor
int docCount = 0;
while (docCount < maxDoc) {
int len = Math.min(MAX_RAW_MERGE_DOCS, maxDoc - docCount);
@@ -354,7 +419,7 @@ public final class Lucene40TermVectorsWr
// NOTE: it's very important to first assign to vectors then pass it to
// termVectorsWriter.addAllDocVectors; see LUCENE-1282
Fields vectors = reader.getTermVectors(docNum);
- addAllDocVectors(vectors, mergeState.fieldInfos);
+ addAllDocVectors(vectors, mergeState);
mergeState.checkAbort.work(300);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene40/package.html Sat Aug 11 19:37:44 2012
@@ -366,7 +366,7 @@ the {@link org.apache.lucene.codecs.Code
factors need no longer be a single byte, they can be any DocValues
{@link org.apache.lucene.index.DocValues.Type type}. Terms need not be unicode
strings, they can be any byte sequence. Term offsets can optionally be indexed
-into the postings lists.</li>
+into the postings lists. Payloads can be stored in the term vectors.</li>
</ul>
<a name="Limitations" id="Limitations"></a>
<h2>Limitations</h2>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java Sat Aug 11 19:37:44 2012
@@ -644,6 +644,11 @@ public class DirectPostingsFormat extend
public boolean hasPositions() {
return hasPos;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
private final class DirectTermsEnum extends TermsEnum {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java Sat Aug 11 19:37:44 2012
@@ -844,6 +844,11 @@ public class MemoryPostingsFormat extend
public boolean hasPositions() {
return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return field.hasPayloads();
+ }
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Sat Aug 11 19:37:44 2012
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
@@ -498,7 +499,7 @@ class SimpleTextFieldsReader extends Fie
private class SimpleTextTerms extends Terms {
private final long termsStart;
- private final IndexOptions indexOptions;
+ private final FieldInfo fieldInfo;
private long sumTotalTermFreq;
private long sumDocFreq;
private int docCount;
@@ -509,7 +510,7 @@ class SimpleTextFieldsReader extends Fie
public SimpleTextTerms(String field, long termsStart) throws IOException {
this.termsStart = termsStart;
- indexOptions = fieldInfos.fieldInfo(field).getIndexOptions();
+ fieldInfo = fieldInfos.fieldInfo(field);
loadTerms();
}
@@ -579,7 +580,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public TermsEnum iterator(TermsEnum reuse) throws IOException {
if (fst != null) {
- return new SimpleTextTermsEnum(fst, indexOptions);
+ return new SimpleTextTermsEnum(fst, fieldInfo.getIndexOptions());
} else {
return TermsEnum.EMPTY;
}
@@ -597,7 +598,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public long getSumTotalTermFreq() {
- return indexOptions == IndexOptions.DOCS_ONLY ? -1 : sumTotalTermFreq;
+ return fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY ? -1 : sumTotalTermFreq;
}
@Override
@@ -612,12 +613,17 @@ class SimpleTextFieldsReader extends Fie
@Override
public boolean hasOffsets() {
- return indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
}
@Override
public boolean hasPositions() {
- return indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return fieldInfo.hasPayloads();
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Sat Aug 11 19:37:44 2012
@@ -127,10 +127,14 @@ public class SimpleTextTermVectorsReader
boolean offsets = Boolean.parseBoolean(readString(FIELDOFFSETS.length, scratch));
readLine();
+ assert StringHelper.startsWith(scratch, FIELDPAYLOADS);
+ boolean payloads = Boolean.parseBoolean(readString(FIELDPAYLOADS.length, scratch));
+
+ readLine();
assert StringHelper.startsWith(scratch, FIELDTERMCOUNT);
int termCount = parseIntAt(FIELDTERMCOUNT.length);
- SimpleTVTerms terms = new SimpleTVTerms(offsets, positions);
+ SimpleTVTerms terms = new SimpleTVTerms(offsets, positions, payloads);
fields.put(fieldName, terms);
for (int j = 0; j < termCount; j++) {
@@ -152,6 +156,9 @@ public class SimpleTextTermVectorsReader
if (positions || offsets) {
if (positions) {
postings.positions = new int[postings.freq];
+ if (payloads) {
+ postings.payloads = new BytesRef[postings.freq];
+ }
}
if (offsets) {
@@ -164,6 +171,17 @@ public class SimpleTextTermVectorsReader
readLine();
assert StringHelper.startsWith(scratch, POSITION);
postings.positions[k] = parseIntAt(POSITION.length);
+ if (payloads) {
+ readLine();
+ assert StringHelper.startsWith(scratch, PAYLOAD);
+ if (scratch.length - PAYLOAD.length == 0) {
+ postings.payloads[k] = null;
+ } else {
+ byte payloadBytes[] = new byte[scratch.length - PAYLOAD.length];
+ System.arraycopy(scratch.bytes, scratch.offset+PAYLOAD.length, payloadBytes, 0, payloadBytes.length);
+ postings.payloads[k] = new BytesRef(payloadBytes);
+ }
+ }
}
if (offsets) {
@@ -259,10 +277,12 @@ public class SimpleTextTermVectorsReader
final SortedMap<BytesRef,SimpleTVPostings> terms;
final boolean hasOffsets;
final boolean hasPositions;
+ final boolean hasPayloads;
- SimpleTVTerms(boolean hasOffsets, boolean hasPositions) {
+ SimpleTVTerms(boolean hasOffsets, boolean hasPositions, boolean hasPayloads) {
this.hasOffsets = hasOffsets;
this.hasPositions = hasPositions;
+ this.hasPayloads = hasPayloads;
terms = new TreeMap<BytesRef,SimpleTVPostings>();
}
@@ -306,6 +326,11 @@ public class SimpleTextTermVectorsReader
public boolean hasPositions() {
return hasPositions;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
}
private static class SimpleTVPostings {
@@ -313,6 +338,7 @@ public class SimpleTextTermVectorsReader
private int positions[];
private int startOffsets[];
private int endOffsets[];
+ private BytesRef payloads[];
}
private static class SimpleTVTermsEnum extends TermsEnum {
@@ -386,7 +412,7 @@ public class SimpleTextTermVectorsReader
}
// TODO: reuse
SimpleTVDocsAndPositionsEnum e = new SimpleTVDocsAndPositionsEnum();
- e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets);
+ e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets, postings.payloads);
return e;
}
@@ -447,6 +473,7 @@ public class SimpleTextTermVectorsReader
private int nextPos;
private Bits liveDocs;
private int[] positions;
+ private BytesRef[] payloads;
private int[] startOffsets;
private int[] endOffsets;
@@ -484,11 +511,12 @@ public class SimpleTextTermVectorsReader
}
}
- public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets) {
+ public void reset(Bits liveDocs, int[] positions, int[] startOffsets, int[] endOffsets, BytesRef payloads[]) {
this.liveDocs = liveDocs;
this.positions = positions;
this.startOffsets = startOffsets;
this.endOffsets = endOffsets;
+ this.payloads = payloads;
this.doc = -1;
didNext = false;
nextPos = 0;
@@ -496,12 +524,13 @@ public class SimpleTextTermVectorsReader
@Override
public BytesRef getPayload() {
- return null;
+ // assert hasPayload(); // you should have called this
+ return payloads == null ? null : payloads[nextPos-1];
}
@Override
public boolean hasPayload() {
- return false;
+ return payloads != null && payloads[nextPos-1] != null;
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsWriter.java Sat Aug 11 19:37:44 2012
@@ -45,10 +45,12 @@ public class SimpleTextTermVectorsWriter
static final BytesRef FIELDNAME = new BytesRef(" name ");
static final BytesRef FIELDPOSITIONS = new BytesRef(" positions ");
static final BytesRef FIELDOFFSETS = new BytesRef(" offsets ");
+ static final BytesRef FIELDPAYLOADS = new BytesRef(" payloads ");
static final BytesRef FIELDTERMCOUNT = new BytesRef(" numterms ");
static final BytesRef TERMTEXT = new BytesRef(" term ");
static final BytesRef TERMFREQ = new BytesRef(" freq ");
static final BytesRef POSITION = new BytesRef(" position ");
+ static final BytesRef PAYLOAD = new BytesRef(" payload ");
static final BytesRef STARTOFFSET = new BytesRef(" startoffset ");
static final BytesRef ENDOFFSET = new BytesRef(" endoffset ");
@@ -61,6 +63,7 @@ public class SimpleTextTermVectorsWriter
private final BytesRef scratch = new BytesRef();
private boolean offsets;
private boolean positions;
+ private boolean payloads;
public SimpleTextTermVectorsWriter(Directory directory, String segment, IOContext context) throws IOException {
this.directory = directory;
@@ -89,7 +92,7 @@ public class SimpleTextTermVectorsWriter
}
@Override
- public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
+ public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
write(FIELD);
write(Integer.toString(info.number));
newLine();
@@ -106,12 +109,17 @@ public class SimpleTextTermVectorsWriter
write(Boolean.toString(offsets));
newLine();
+ write(FIELDPAYLOADS);
+ write(Boolean.toString(payloads));
+ newLine();
+
write(FIELDTERMCOUNT);
write(Integer.toString(numTerms));
newLine();
this.positions = positions;
this.offsets = offsets;
+ this.payloads = payloads;
}
@Override
@@ -126,13 +134,22 @@ public class SimpleTextTermVectorsWriter
}
@Override
- public void addPosition(int position, int startOffset, int endOffset) throws IOException {
+ public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
assert positions || offsets;
if (positions) {
write(POSITION);
write(Integer.toString(position));
newLine();
+
+ if (payloads) {
+ write(PAYLOAD);
+ if (payload != null) {
+ assert payload.length > 0;
+ write(payload);
+ }
+ newLine();
+ }
}
if (offsets) {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/document/FieldType.java Sat Aug 11 19:37:44 2012
@@ -39,6 +39,7 @@ public class FieldType implements Indexa
private boolean storeTermVectors;
private boolean storeTermVectorOffsets;
private boolean storeTermVectorPositions;
+ private boolean storeTermVectorPayloads;
private boolean omitNorms;
private IndexOptions indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
private DocValues.Type docValueType;
@@ -53,6 +54,7 @@ public class FieldType implements Indexa
this.storeTermVectors = ref.storeTermVectors();
this.storeTermVectorOffsets = ref.storeTermVectorOffsets();
this.storeTermVectorPositions = ref.storeTermVectorPositions();
+ this.storeTermVectorPayloads = ref.storeTermVectorPayloads();
this.omitNorms = ref.omitNorms();
this.indexOptions = ref.indexOptions();
this.docValueType = ref.docValueType();
@@ -132,6 +134,15 @@ public class FieldType implements Indexa
this.storeTermVectorPositions = value;
}
+ public boolean storeTermVectorPayloads() {
+ return this.storeTermVectorPayloads;
+ }
+
+ public void setStoreTermVectorPayloads(boolean value) {
+ checkIfFrozen();
+ this.storeTermVectorPayloads = value;
+ }
+
public boolean omitNorms() {
return this.omitNorms;
}
@@ -198,24 +209,19 @@ public class FieldType implements Indexa
result.append(",");
result.append("indexed");
if (tokenized()) {
- if (result.length() > 0)
- result.append(",");
- result.append("tokenized");
+ result.append(",tokenized");
}
if (storeTermVectors()) {
- if (result.length() > 0)
- result.append(",");
- result.append("termVector");
+ result.append(",termVector");
}
if (storeTermVectorOffsets()) {
- if (result.length() > 0)
- result.append(",");
- result.append("termVectorOffsets");
+ result.append(",termVectorOffsets");
}
if (storeTermVectorPositions()) {
- if (result.length() > 0)
- result.append(",");
- result.append("termVectorPosition");
+ result.append(",termVectorPosition");
+ if (storeTermVectorPayloads()) {
+ result.append(",termVectorPayloads");
+ }
}
if (omitNorms()) {
result.append(",omitNorms");
@@ -232,7 +238,9 @@ public class FieldType implements Indexa
}
}
if (docValueType != null) {
- result.append(",docValueType=");
+ if (result.length() > 0)
+ result.append(",");
+ result.append("docValueType=");
result.append(docValueType);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Sat Aug 11 19:37:44 2012
@@ -833,7 +833,10 @@ public class CheckIndex {
}
lastPos = pos;
if (postings.hasPayload()) {
- postings.getPayload();
+ BytesRef payload = postings.getPayload();
+ if (payload.length < 1) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " payload length is out of bounds " + payload.length);
+ }
}
if (hasOffsets) {
int startOffset = postings.startOffset();
@@ -1420,6 +1423,8 @@ public class CheckIndex {
Terms terms = tfv.terms(field);
termsEnum = terms.iterator(termsEnum);
final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ final boolean postingsHasPayload = fieldInfo.hasPayloads();
+ final boolean vectorsHasPayload = terms.hasPayloads();
Terms postingsTerms = postingsFields.terms(field);
if (postingsTerms == null) {
@@ -1522,6 +1527,34 @@ public class CheckIndex {
throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
}
}
+
+ BytesRef payload = null;
+ if (postings.hasPayload()) {
+ assert vectorsHasPayload;
+ payload = postings.getPayload();
+ }
+
+ if (postingsHasPayload && vectorsHasPayload) {
+ assert postingsPostings != null;
+
+ if (payload == null) {
+ // we have payloads, but not at this position.
+ // postings has payloads too, it should not have one at this position
+ if (postingsPostings.hasPayload()) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has no payload but postings does: " + postingsPostings.getPayload());
+ }
+ } else {
+ // we have payloads, and one at this position
+ // postings should also have one at this position, with the same bytes.
+ if (!postingsPostings.hasPayload()) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but postings does not.");
+ }
+ BytesRef postingsPayload = postingsPostings.getPayload();
+ if (!payload.equals(postingsPayload)) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + " has payload=" + payload + " but differs from postings payload=" + postingsPayload);
+ }
+ }
+ }
}
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Sat Aug 11 19:37:44 2012
@@ -119,6 +119,11 @@ public class FilterAtomicReader extends
public boolean hasPositions() {
return in.hasPositions();
}
+
+ @Override
+ public boolean hasPayloads() {
+ return in.hasPayloads();
+ }
}
/** Base class for filtering {@link TermsEnum} implementations. */
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Sat Aug 11 19:37:44 2012
@@ -42,6 +42,9 @@ public interface IndexableFieldType {
/** True if term vector positions should be indexed */
public boolean storeTermVectorPositions();
+
+ /** True if term vector payloads should be indexed */
+ public boolean storeTermVectorPayloads();
/** True if norms should not be indexed */
public boolean omitNorms();
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MergeState.java Sat Aug 11 19:37:44 2012
@@ -199,6 +199,7 @@ public class MergeState {
// and we could make a codec(wrapper) to do all of this privately so IW is uninvolved
public PayloadProcessorProvider payloadProcessorProvider;
public ReaderPayloadProcessor[] readerPayloadProcessor;
+ public ReaderPayloadProcessor currentReaderPayloadProcessor;
public PayloadProcessor[] currentPayloadProcessor;
// TODO: get rid of this? it tells you which segments are 'aligned' (e.g. for bulk merging)
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MultiTerms.java Sat Aug 11 19:37:44 2012
@@ -39,6 +39,7 @@ public final class MultiTerms extends Te
private final Comparator<BytesRef> termComp;
private final boolean hasOffsets;
private final boolean hasPositions;
+ private final boolean hasPayloads;
public MultiTerms(Terms[] subs, ReaderSlice[] subSlices) throws IOException {
this.subs = subs;
@@ -48,6 +49,7 @@ public final class MultiTerms extends Te
assert subs.length > 0 : "inefficient: don't use MultiTerms over one sub";
boolean _hasOffsets = true;
boolean _hasPositions = true;
+ boolean _hasPayloads = false;
for(int i=0;i<subs.length;i++) {
if (_termComp == null) {
_termComp = subs[i].getComparator();
@@ -61,11 +63,13 @@ public final class MultiTerms extends Te
}
_hasOffsets &= subs[i].hasOffsets();
_hasPositions &= subs[i].hasPositions();
+ _hasPayloads |= subs[i].hasPayloads();
}
termComp = _termComp;
hasOffsets = _hasOffsets;
hasPositions = _hasPositions;
+ hasPayloads = hasPositions && _hasPayloads; // if all subs have pos, and at least one has payloads.
}
@Override
@@ -161,5 +165,10 @@ public final class MultiTerms extends Te
public boolean hasPositions() {
return hasPositions;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return hasPayloads;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Sat Aug 11 19:37:44 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
@@ -36,9 +37,12 @@ final class TermVectorsConsumerPerField
boolean doVectors;
boolean doVectorPositions;
boolean doVectorOffsets;
+ boolean doVectorPayloads;
int maxNumPostings;
OffsetAttribute offsetAttribute;
+ PayloadAttribute payloadAttribute;
+ boolean hasPayloads; // if enabled, and we actually saw any for this field
public TermVectorsConsumerPerField(TermsHashPerField termsHashPerField, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
this.termsHashPerField = termsHashPerField;
@@ -58,6 +62,8 @@ final class TermVectorsConsumerPerField
doVectors = false;
doVectorPositions = false;
doVectorOffsets = false;
+ doVectorPayloads = false;
+ hasPayloads = false;
for(int i=0;i<count;i++) {
IndexableField field = fields[i];
@@ -65,6 +71,12 @@ final class TermVectorsConsumerPerField
doVectors = true;
doVectorPositions |= field.fieldType().storeTermVectorPositions();
doVectorOffsets |= field.fieldType().storeTermVectorOffsets();
+ if (doVectorPositions) {
+ doVectorPayloads |= field.fieldType().storeTermVectorPayloads();
+ } else if (field.fieldType().storeTermVectorPayloads()) {
+ // TODO: move this check somewhere else, and impl the other missing ones
+ throw new IllegalArgumentException("cannot index term vector payloads for field: " + field + " without term vector positions");
+ }
}
}
@@ -121,7 +133,7 @@ final class TermVectorsConsumerPerField
final int[] termIDs = termsHashPerField.sortPostings(tv.getComparator());
- tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets);
+ tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
final ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null;
final ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null;
@@ -165,52 +177,64 @@ final class TermVectorsConsumerPerField
} else {
offsetAttribute = null;
}
+ if (doVectorPayloads && fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
+ payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
+ } else {
+ payloadAttribute = null;
+ }
}
-
- @Override
- void newTerm(final int termID) {
- assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
-
- postings.freqs[termID] = 1;
-
+
+ void writeProx(TermVectorsPostingsArray postings, int termID) {
if (doVectorOffsets) {
int startOffset = fieldState.offset + offsetAttribute.startOffset();
int endOffset = fieldState.offset + offsetAttribute.endOffset();
- termsHashPerField.writeVInt(1, startOffset);
+ termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
termsHashPerField.writeVInt(1, endOffset - startOffset);
postings.lastOffsets[termID] = endOffset;
}
if (doVectorPositions) {
- termsHashPerField.writeVInt(0, fieldState.position);
+ final BytesRef payload;
+ if (payloadAttribute == null) {
+ payload = null;
+ } else {
+ payload = payloadAttribute.getPayload();
+ }
+
+ final int pos = fieldState.position - postings.lastPositions[termID];
+ if (payload != null && payload.length > 0) {
+ termsHashPerField.writeVInt(0, (pos<<1)|1);
+ termsHashPerField.writeVInt(0, payload.length);
+ termsHashPerField.writeBytes(0, payload.bytes, payload.offset, payload.length);
+ hasPayloads = true;
+ } else {
+ termsHashPerField.writeVInt(0, pos<<1);
+ }
postings.lastPositions[termID] = fieldState.position;
}
}
@Override
- void addTerm(final int termID) {
+ void newTerm(final int termID) {
+ assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
+ TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
- assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
+ postings.freqs[termID] = 1;
+ postings.lastOffsets[termID] = 0;
+ postings.lastPositions[termID] = 0;
+
+ writeProx(postings, termID);
+ }
+ @Override
+ void addTerm(final int termID) {
+ assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
postings.freqs[termID]++;
- if (doVectorOffsets) {
- int startOffset = fieldState.offset + offsetAttribute.startOffset();
- int endOffset = fieldState.offset + offsetAttribute.endOffset();
-
- termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
- postings.lastOffsets[termID] = endOffset;
- }
-
- if (doVectorPositions) {
- termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID]);
- postings.lastPositions[termID] = fieldState.position;
- }
+ writeProx(postings, termID);
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/Terms.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/Terms.java Sat Aug 11 19:37:44 2012
@@ -110,6 +110,9 @@ public abstract class Terms {
/** Returns true if documents in this field store positions. */
public abstract boolean hasPositions();
+
+ /** Returns true if documents in this field store payloads. */
+ public abstract boolean hasPayloads();
public final static Terms[] EMPTY_ARRAY = new Terms[0];
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sat Aug 11 19:37:44 2012
@@ -209,6 +209,7 @@ public class TestDuelingCodecs extends L
assertTermsStatistics(leftTerms, rightTerms);
assertEquals(leftTerms.hasOffsets(), rightTerms.hasOffsets());
assertEquals(leftTerms.hasPositions(), rightTerms.hasPositions());
+ assertEquals(leftTerms.hasPayloads(), rightTerms.hasPayloads());
TermsEnum leftTermsEnum = leftTerms.iterator(null);
TermsEnum rightTermsEnum = rightTerms.iterator(null);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java Sat Aug 11 19:37:44 2012
@@ -72,6 +72,15 @@ public class TestIndexableField extends
public boolean storeTermVectorPositions() {
return counter % 2 == 1 && counter % 10 != 9;
}
+
+ @Override
+ public boolean storeTermVectorPayloads() {
+ if (PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ return false; // 3.x doesnt support
+ } else {
+ return counter % 2 == 1 && counter % 10 != 9;
+ }
+ }
@Override
public boolean omitNorms() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java Sat Aug 11 19:37:44 2012
@@ -145,8 +145,18 @@ public class TestPayloadProcessorProvide
Document doc = new Document();
doc.add(newField("id", "doc" + i, customType));
doc.add(newTextField("content", "doc content " + i, Field.Store.NO));
- doc.add(new TextField("p", payloadTS1));
- doc.add(new TextField("p", payloadTS2));
+ if (PREFLEX_IMPERSONATION_IS_ACTIVE || random.nextBoolean()) {
+ doc.add(new TextField("p", payloadTS1));
+ doc.add(new TextField("p", payloadTS2));
+ } else {
+ FieldType type = new FieldType(TextField.TYPE_NOT_STORED);
+ type.setStoreTermVectors(true);
+ type.setStoreTermVectorPositions(true);
+ type.setStoreTermVectorPayloads(true);
+ type.setStoreTermVectorOffsets(random.nextBoolean());
+ doc.add(new Field("p", payloadTS1, type));
+ doc.add(new Field("p", payloadTS2, type));
+ }
writer.addDocument(doc);
if (multipleCommits && (i % 4 == 0)) {
writer.commit();
Copied: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (from r1371996, lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java?p2=lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java&p1=lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java&r1=1371996&r2=1372010&rev=1372010&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java Sat Aug 11 19:37:44 2012
@@ -32,7 +32,9 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
+@SuppressCodecs("Lucene3x")
public class TestPayloadsOnVectors extends LuceneTestCase {
/** some docs have payload att, some not */
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/search/FieldCacheRewriteMethod.java Sat Aug 11 19:37:44 2012
@@ -132,6 +132,11 @@ public final class FieldCacheRewriteMeth
public boolean hasPositions() {
return false;
}
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
});
assert termsEnum != null;
Modified: lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/branch_4x/lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sat Aug 11 19:37:44 2012
@@ -787,7 +787,12 @@ public class MemoryIndex {
@Override
public boolean hasPositions() {
return true;
- }
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return false;
+ }
};
}
}
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sat Aug 11 19:37:44 2012
@@ -486,6 +486,9 @@ public abstract class BaseTokenStreamTes
ft.setStoreTermVectors(true);
ft.setStoreTermVectorOffsets(random.nextBoolean());
ft.setStoreTermVectorPositions(random.nextBoolean());
+ if (ft.storeTermVectorPositions() && !PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ ft.setStoreTermVectorPayloads(random.nextBoolean());
+ }
}
if (random.nextBoolean()) {
ft.setOmitNorms(true);
Modified: lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java?rev=1372010&r1=1372009&r2=1372010&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/test-framework/src/java/org/apache/lucene/codecs/lucene3x/PreFlexRWTermVectorsWriter.java Sat Aug 11 19:37:44 2012
@@ -75,9 +75,12 @@ final class PreFlexRWTermVectorsWriter e
private String lastFieldName;
@Override
- public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets) throws IOException {
+ public void startField(FieldInfo info, int numTerms, boolean positions, boolean offsets, boolean payloads) throws IOException {
assert lastFieldName == null || info.name.compareTo(lastFieldName) > 0: "fieldName=" + info.name + " lastFieldName=" + lastFieldName;
lastFieldName = info.name;
+ if (payloads) {
+ throw new UnsupportedOperationException("3.x codec does not support payloads on vectors!");
+ }
this.positions = positions;
this.offsets = offsets;
lastTerm.length = 0;
@@ -136,24 +139,8 @@ final class PreFlexRWTermVectorsWriter e
int lastOffset = 0;
@Override
- public void addProx(int numProx, DataInput positions, DataInput offsets) throws IOException {
- // TODO: technically we could just copy bytes and not re-encode if we knew the length...
- if (positions != null) {
- for (int i = 0; i < numProx; i++) {
- tvf.writeVInt(positions.readVInt());
- }
- }
-
- if (offsets != null) {
- for (int i = 0; i < numProx; i++) {
- tvf.writeVInt(offsets.readVInt());
- tvf.writeVInt(offsets.readVInt());
- }
- }
- }
-
- @Override
- public void addPosition(int position, int startOffset, int endOffset) throws IOException {
+ public void addPosition(int position, int startOffset, int endOffset, BytesRef payload) throws IOException {
+ assert payload == null;
if (positions && offsets) {
// write position delta
tvf.writeVInt(position - lastPosition);