You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/04/29 19:18:44 UTC
svn commit: r1591025 [1/2] - in /lucene/dev/trunk: ./ lucene/ lucene/codecs/
lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/ lucene/core/
lucene/core/src/java/org/apache/lucene/analysis/
lucene/core/src/java/org/apache/lucene/codecs/ lucene...
Author: mikemccand
Date: Tue Apr 29 17:18:41 2014
New Revision: 1591025
URL: http://svn.apache.org/r1591025
Log:
LUCENE-5611: simplify the default indexing chain
Added:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
- copied unchanged from r1591009, lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
Removed:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java
Modified:
lucene/dev/trunk/ (props changed)
lucene/dev/trunk/lucene/ (props changed)
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/codecs/ (props changed)
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
lucene/dev/trunk/lucene/core/ (props changed)
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMDirectory.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/store/RAMOutputStream.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/QueryBuilder.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/document/TestDocument.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/store/TestHugeRamFile.java
lucene/dev/trunk/lucene/core/src/test/org/apache/lucene/util/TestAttributeSource.java
lucene/dev/trunk/lucene/misc/ (props changed)
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/index/sorter/SortingAtomicReader.java
lucene/dev/trunk/lucene/test-framework/ (props changed)
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/analysis/ValidatingTokenFilter.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/trunk/solr/ (props changed)
lucene/dev/trunk/solr/core/ (props changed)
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/JsonPreAnalyzedParser.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/schema/SimplePreAnalyzedParser.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Tue Apr 29 17:18:41 2014
@@ -77,6 +77,13 @@ Other
======================= Lucene 4.9.0 =======================
+Changes in Runtime Behavior
+
+* LUCENE-5611: Changing the term vector options for multiple field
+ instances by the same name in one document is not longer accepted;
+ IndexWriter will now throw IllegalArgumentException. (Robert Muir,
+ Mike McCandless)
+
New Features
* LUCENE-5610: Add Terms.getMin and Terms.getMax to get the lowest and
@@ -93,6 +100,13 @@ API Changes
* LUCENE-5621: Deprecate IndexOutput.flush: this is not used by Lucene.
(Robert Muir)
+* LUCENE-5611: Simplified Lucene's default indexing chain / APIs.
+ AttributeSource/TokenStream.getAttribute now returns null if the
+ attribute is not present (previously it threw
+ IllegalArgumentException). StoredFieldsWriter.startDocument no
+ longer receives the number of fields that will be added (Robert
+ Muir, Mike McCandless)
+
Optimizations
* LUCENE-5603: hunspell stemmer more efficiently strips prefixes
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java Tue Apr 29 17:18:41 2014
@@ -98,13 +98,12 @@ public class SimpleTextStoredFieldsReade
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
in.seek(offsets[n]);
- readLine();
- assert StringHelper.startsWith(scratch, NUM);
- int numFields = parseIntAt(NUM.length);
- for (int i = 0; i < numFields; i++) {
+ while (true) {
readLine();
- assert StringHelper.startsWith(scratch, FIELD);
+ if (StringHelper.startsWith(scratch, FIELD) == false) {
+ break;
+ }
int fieldNumber = parseIntAt(FIELD.length);
FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
readLine();
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java Tue Apr 29 17:18:41 2014
@@ -53,7 +53,6 @@ public class SimpleTextStoredFieldsWrite
final static BytesRef END = new BytesRef("END");
final static BytesRef DOC = new BytesRef("doc ");
- final static BytesRef NUM = new BytesRef(" numfields ");
final static BytesRef FIELD = new BytesRef(" field ");
final static BytesRef NAME = new BytesRef(" name ");
final static BytesRef TYPE = new BytesRef(" type ");
@@ -76,15 +75,11 @@ public class SimpleTextStoredFieldsWrite
}
@Override
- public void startDocument(int numStoredFields) throws IOException {
+ public void startDocument() throws IOException {
write(DOC);
write(Integer.toString(numDocsWritten));
newLine();
- write(NUM);
- write(Integer.toString(numStoredFields));
- newLine();
-
numDocsWritten++;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java Tue Apr 29 17:18:41 2014
@@ -171,8 +171,9 @@ public abstract class TokenStream extend
*/
public void end() throws IOException {
clearAttributes(); // LUCENE-3849: don't consume dirty atts
- if (hasAttribute(PositionIncrementAttribute.class)) {
- getAttribute(PositionIncrementAttribute.class).setPositionIncrement(0);
+ PositionIncrementAttribute posIncAtt = getAttribute(PositionIncrementAttribute.class);
+ if (posIncAtt != null) {
+ posIncAtt.setPositionIncrement(0);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Tue Apr 29 17:18:41 2014
@@ -31,8 +31,8 @@ import org.apache.lucene.index.AtomicRea
* Codec API for writing stored fields:
* <p>
* <ol>
- * <li>For every document, {@link #startDocument(int)} is called,
- * informing the Codec how many fields will be written.
+ * <li>For every document, {@link #startDocument()} is called,
+ * informing the Codec that a new document has started.
* <li>{@link #writeField(FieldInfo, StorableField)} is called for
* each field in the document.
* <li>After all documents have been written, {@link #finish(FieldInfos, int)}
@@ -51,10 +51,9 @@ public abstract class StoredFieldsWriter
/** Called before writing the stored fields of the document.
* {@link #writeField(FieldInfo, StorableField)} will be called
- * <code>numStoredFields</code> times. Note that this is
- * called even if the document has no stored fields, in
- * this case <code>numStoredFields</code> will be zero. */
- public abstract void startDocument(int numStoredFields) throws IOException;
+ * for each stored field. Note that this is
+ * called even if the document has no stored fields. */
+ public abstract void startDocument() throws IOException;
/** Called when a document and all its fields have been added. */
public void finishDocument() throws IOException {}
@@ -69,14 +68,14 @@ public abstract class StoredFieldsWriter
/** Called before {@link #close()}, passing in the number
* of documents that were written. Note that this is
* intentionally redundant (equivalent to the number of
- * calls to {@link #startDocument(int)}, but a Codec should
+ * calls to {@link #startDocument()}, but a Codec should
* check that this is the case to detect the JRE bug described
* in LUCENE-1282. */
public abstract void finish(FieldInfos fis, int numDocs) throws IOException;
/** Merges in the stored fields from the readers in
* <code>mergeState</code>. The default implementation skips
- * over deleted documents, and uses {@link #startDocument(int)},
+ * over deleted documents, and uses {@link #startDocument()},
* {@link #writeField(FieldInfo, StorableField)}, and {@link #finish(FieldInfos, int)},
* returning the number of documents that were written.
* Implementations can override this method for more sophisticated
@@ -109,12 +108,7 @@ public abstract class StoredFieldsWriter
/** sugar method for startDocument() + writeField() for every stored field in the document */
protected final void addDocument(Iterable<? extends StorableField> doc, FieldInfos fieldInfos) throws IOException {
- int storedCount = 0;
- for (StorableField field : doc) {
- storedCount++;
- }
-
- startDocument(storedCount);
+ startDocument();
for (StorableField field : doc) {
writeField(fieldInfos.fieldInfo(field.name()), field);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Tue Apr 29 17:18:41 2014
@@ -145,20 +145,23 @@ public final class CompressingStoredFiel
}
}
+ private int numStoredFieldsInDoc;
+
+ @Override
+ public void startDocument() throws IOException {
+ }
+
@Override
- public void startDocument(int numStoredFields) throws IOException {
+ public void finishDocument() throws IOException {
if (numBufferedDocs == this.numStoredFields.length) {
final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);
this.numStoredFields = Arrays.copyOf(this.numStoredFields, newLength);
endOffsets = Arrays.copyOf(endOffsets, newLength);
}
- this.numStoredFields[numBufferedDocs] = numStoredFields;
+ this.numStoredFields[numBufferedDocs] = numStoredFieldsInDoc;
+ numStoredFieldsInDoc = 0;
+ endOffsets[numBufferedDocs] = bufferedDocs.length;
++numBufferedDocs;
- }
-
- @Override
- public void finishDocument() throws IOException {
- endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
if (triggerFlush()) {
flush();
}
@@ -242,6 +245,9 @@ public final class CompressingStoredFiel
@Override
public void writeField(FieldInfo info, StorableField field)
throws IOException {
+
+ ++numStoredFieldsInDoc;
+
int bits = 0;
final BytesRef bytes;
final String string;
@@ -391,8 +397,9 @@ public final class CompressingStoredFiel
// copy non-deleted docs
for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
final int diff = docID - it.docBase;
- startDocument(it.numStoredFields[diff]);
+ startDocument();
bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+ numStoredFieldsInDoc = it.numStoredFields[diff];
finishDocument();
++docCount;
mergeState.checkAbort.work(300);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java Tue Apr 29 17:18:41 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -85,6 +86,7 @@ public final class Lucene40StoredFieldsW
private final String segment;
private IndexOutput fieldsStream;
private IndexOutput indexStream;
+ private final RAMOutputStream fieldsBuffer = new RAMOutputStream();
/** Sole constructor. */
public Lucene40StoredFieldsWriter(Directory directory, String segment, IOContext context) throws IOException {
@@ -109,14 +111,23 @@ public final class Lucene40StoredFieldsW
}
}
+ int numStoredFields;
+
// Writes the contents of buffer into the fields stream
// and adds a new entry for this document into the index
// stream. This assumes the buffer was already written
// in the correct fields format.
@Override
- public void startDocument(int numStoredFields) throws IOException {
+ public void startDocument() throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
+ }
+
+ @Override
+ public void finishDocument() throws IOException {
fieldsStream.writeVInt(numStoredFields);
+ fieldsBuffer.writeTo(fieldsStream);
+ fieldsBuffer.reset();
+ numStoredFields = 0;
}
@Override
@@ -140,7 +151,9 @@ public final class Lucene40StoredFieldsW
@Override
public void writeField(FieldInfo info, StorableField field) throws IOException {
- fieldsStream.writeVInt(info.number);
+ numStoredFields++;
+
+ fieldsBuffer.writeVInt(info.number);
int bits = 0;
final BytesRef bytes;
final String string;
@@ -177,22 +190,22 @@ public final class Lucene40StoredFieldsW
}
}
- fieldsStream.writeByte((byte) bits);
+ fieldsBuffer.writeByte((byte) bits);
if (bytes != null) {
- fieldsStream.writeVInt(bytes.length);
- fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ fieldsBuffer.writeVInt(bytes.length);
+ fieldsBuffer.writeBytes(bytes.bytes, bytes.offset, bytes.length);
} else if (string != null) {
- fieldsStream.writeString(field.stringValue());
+ fieldsBuffer.writeString(field.stringValue());
} else {
if (number instanceof Byte || number instanceof Short || number instanceof Integer) {
- fieldsStream.writeInt(number.intValue());
+ fieldsBuffer.writeInt(number.intValue());
} else if (number instanceof Long) {
- fieldsStream.writeLong(number.longValue());
+ fieldsBuffer.writeLong(number.longValue());
} else if (number instanceof Float) {
- fieldsStream.writeInt(Float.floatToIntBits(number.floatValue()));
+ fieldsBuffer.writeInt(Float.floatToIntBits(number.floatValue()));
} else if (number instanceof Double) {
- fieldsStream.writeLong(Double.doubleToLongBits(number.doubleValue()));
+ fieldsBuffer.writeLong(Double.doubleToLongBits(number.doubleValue()));
} else {
throw new AssertionError("Cannot get here");
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41SkipWriter.java Tue Apr 29 17:18:41 2014
@@ -88,20 +88,43 @@ final class Lucene41SkipWriter extends M
this.fieldHasOffsets = fieldHasOffsets;
this.fieldHasPayloads = fieldHasPayloads;
}
+
+ // tricky: we only skip data for blocks (terms with more than 128 docs), but re-init'ing the skipper
+ // is pretty slow for rare terms in large segments as we have to fill O(log #docs in segment) of junk.
+ // this is the vast majority of terms (worst case: ID field or similar). so in resetSkip() we save
+ // away the previous pointers, and lazy-init only if we need to buffer skip data for the term.
+ private boolean initialized;
+ long lastDocFP;
+ long lastPosFP;
+ long lastPayFP;
@Override
public void resetSkip() {
- super.resetSkip();
- Arrays.fill(lastSkipDoc, 0);
- Arrays.fill(lastSkipDocPointer, docOut.getFilePointer());
+ lastDocFP = docOut.getFilePointer();
if (fieldHasPositions) {
- Arrays.fill(lastSkipPosPointer, posOut.getFilePointer());
- if (fieldHasPayloads) {
- Arrays.fill(lastPayloadByteUpto, 0);
- }
+ lastPosFP = posOut.getFilePointer();
if (fieldHasOffsets || fieldHasPayloads) {
- Arrays.fill(lastSkipPayPointer, payOut.getFilePointer());
+ lastPayFP = payOut.getFilePointer();
+ }
+ }
+ initialized = false;
+ }
+
+ public void initSkip() {
+ if (!initialized) {
+ super.resetSkip();
+ Arrays.fill(lastSkipDoc, 0);
+ Arrays.fill(lastSkipDocPointer, lastDocFP);
+ if (fieldHasPositions) {
+ Arrays.fill(lastSkipPosPointer, lastPosFP);
+ if (fieldHasPayloads) {
+ Arrays.fill(lastPayloadByteUpto, 0);
+ }
+ if (fieldHasOffsets || fieldHasPayloads) {
+ Arrays.fill(lastSkipPayPointer, lastPayFP);
+ }
}
+ initialized = true;
}
}
@@ -109,6 +132,7 @@ final class Lucene41SkipWriter extends M
* Sets the values for the current skip data.
*/
public void bufferSkip(int doc, int numDocs, long posFP, long payFP, int posBufferUpto, int payloadByteUpto) throws IOException {
+ initSkip();
this.curDoc = doc;
this.curDocPointer = docOut.getFilePointer();
this.curPosPointer = posFP;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/Field.java Tue Apr 29 17:18:41 2014
@@ -244,11 +244,6 @@ public class Field implements IndexableF
throw new IllegalArgumentException("it doesn't make sense to have a field that "
+ "is neither indexed nor stored");
}
- if (!type.indexed() && (type.storeTermVectors())) {
- throw new IllegalArgumentException("cannot store term vector information "
- + "for a field that is not indexed");
- }
-
this.type = type;
this.name = name;
this.fieldsData = value;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/document/FieldType.java Tue Apr 29 17:18:41 2014
@@ -382,9 +382,9 @@ public class FieldType implements Indexa
}
if (storeTermVectorPositions()) {
result.append(",termVectorPosition");
- if (storeTermVectorPayloads()) {
- result.append(",termVectorPayloads");
- }
+ }
+ if (storeTermVectorPayloads()) {
+ result.append(",termVectorPayloads");
}
if (omitNorms()) {
result.append(",omitNorms");
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java Tue Apr 29 17:18:41 2014
@@ -121,10 +121,6 @@ class BinaryDocValuesWriter extends DocV
});
}
- @Override
- public void abort() {
- }
-
// iterates over the values we have in ram
private class BytesIterator implements Iterator<BytesRef> {
final BytesRef value = new BytesRef();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java Tue Apr 29 17:18:41 2014
@@ -20,8 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
abstract class DocConsumer {
- abstract void processDocument(FieldInfos.Builder fieldInfos) throws IOException;
- abstract void finishDocument() throws IOException;
+ abstract void processDocument() throws IOException;
abstract void flush(final SegmentWriteState state) throws IOException;
abstract void abort();
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java Tue Apr 29 17:18:41 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
abstract class DocValuesWriter {
- abstract void abort() throws IOException;
abstract void finish(int numDoc);
abstract void flush(SegmentWriteState state, DocValuesConsumer consumer) throws IOException;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Apr 29 17:18:41 2014
@@ -43,19 +43,13 @@ import org.apache.lucene.util.InfoStream
* This class accepts multiple added documents and directly
* writes segment files.
*
- * Each added document is passed to the {@link DocConsumer},
- * which in turn processes the document and interacts with
- * other consumers in the indexing chain. Certain
- * consumers, like {@link StoredFieldsConsumer} and {@link
- * TermVectorsConsumer}, digest a document and
- * immediately write bytes to the "doc store" files (ie,
- * they do not consume RAM per document, except while they
- * are processing the document).
+ * Each added document is passed to the indexing chain,
+ * which in turn processes the document into the different
+ * codec formats. Some formats write bytes to files
+ * immediately, e.g. stored fields and term vectors, while
+ * others are buffered by the indexing chain and written
+ * only on flush.
*
- * Other consumers, eg {@link FreqProxTermsWriter} and
- * {@link NormsConsumer}, buffer bytes in RAM and flush only
- * when a new segment is produced.
-
* Once we have used our allowed RAM buffer, or the number
* of added docs is large enough (in the case we are
* flushing by doc count instead of RAM usage), we create a
@@ -398,7 +392,7 @@ final class DocumentsWriter implements C
return hasEvents;
}
- private final void ensureInitialized(ThreadState state) {
+ private final void ensureInitialized(ThreadState state) throws IOException {
if (state.isActive() && state.dwpt == null) {
final FieldInfos.Builder infos = new FieldInfos.Builder(
writer.globalFieldNumberMap);
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Tue Apr 29 17:18:41 2014
@@ -51,48 +51,15 @@ class DocumentsWriterPerThread {
* documents.
*/
abstract static class IndexingChain {
- abstract DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread);
+ abstract DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) throws IOException;
}
static final IndexingChain defaultIndexingChain = new IndexingChain() {
@Override
- DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) {
- /*
- This is the current indexing chain:
-
- DocConsumer / DocConsumerPerThread
- --> code: DocFieldProcessor
- --> DocFieldConsumer / DocFieldConsumerPerField
- --> code: DocFieldConsumers / DocFieldConsumersPerField
- --> code: DocInverter / DocInverterPerField
- --> InvertedDocConsumer / InvertedDocConsumerPerField
- --> code: TermsHash / TermsHashPerField
- --> TermsHashConsumer / TermsHashConsumerPerField
- --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
- --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
- --> InvertedDocEndConsumer / InvertedDocConsumerPerField
- --> code: NormsConsumer / NormsConsumerPerField
- --> StoredFieldsConsumer
- --> TwoStoredFieldConsumers
- -> code: StoredFieldsProcessor
- -> code: DocValuesProcessor
- */
-
- // Build up indexing chain:
-
- final TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
- final TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
-
- final InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true,
- new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
- final NormsConsumer normsWriter = new NormsConsumer();
- final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
- final StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(
- new StoredFieldsProcessor(documentsWriterPerThread),
- new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
- return new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields);
+ DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) throws IOException {
+ return new DefaultIndexingChain(documentsWriterPerThread);
}
};
@@ -103,7 +70,6 @@ class DocumentsWriterPerThread {
Similarity similarity;
int docID;
IndexDocument doc;
- String maxTermPrefix;
DocState(DocumentsWriterPerThread docWriter, InfoStream infoStream) {
this.docWriter = docWriter;
@@ -180,7 +146,7 @@ class DocumentsWriterPerThread {
boolean aborting = false; // True if an abort is pending
boolean hasAborted = false; // True if the last exception throws by #updateDocument was aborting
- private FieldInfos.Builder fieldInfos;
+ private final FieldInfos.Builder fieldInfos;
private final InfoStream infoStream;
private int numDocsInRAM;
final DocumentsWriterDeleteQueue deleteQueue;
@@ -192,7 +158,7 @@ class DocumentsWriterPerThread {
public DocumentsWriterPerThread(String segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue,
- FieldInfos.Builder fieldInfos) {
+ FieldInfos.Builder fieldInfos) throws IOException {
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
@@ -218,13 +184,16 @@ class DocumentsWriterPerThread {
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.getIndexingChain().getChain(this);
-
}
void setAborting() {
aborting = true;
}
+ public FieldInfos.Builder getFieldInfosBuilder() {
+ return fieldInfos;
+ }
+
boolean checkAndResetHasAborted() {
final boolean retval = hasAborted;
hasAborted = false;
@@ -250,7 +219,7 @@ class DocumentsWriterPerThread {
boolean success = false;
try {
try {
- consumer.processDocument(fieldInfos);
+ consumer.processDocument();
} finally {
docState.clear();
}
@@ -266,15 +235,6 @@ class DocumentsWriterPerThread {
}
}
}
- success = false;
- try {
- consumer.finishDocument();
- success = true;
- } finally {
- if (!success) {
- abort(filesToDelete);
- }
- }
finishDocument(delTerm);
}
@@ -296,7 +256,7 @@ class DocumentsWriterPerThread {
boolean success = false;
try {
- consumer.processDocument(fieldInfos);
+ consumer.processDocument();
success = true;
} finally {
if (!success) {
@@ -310,16 +270,6 @@ class DocumentsWriterPerThread {
}
}
}
- success = false;
- try {
- consumer.finishDocument();
- success = true;
- } finally {
- if (!success) {
- abort(filesToDelete);
- }
- }
-
finishDocument(null);
}
allDocsIndexed = true;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Tue Apr 29 17:18:41 2014
@@ -169,16 +169,10 @@ public final class FieldInfo {
// should only be called by FieldInfos#addOrUpdate
void update(boolean indexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
//System.out.println("FI.update field=" + name + " indexed=" + indexed + " omitNorms=" + omitNorms + " this.omitNorms=" + this.omitNorms);
- if (this.indexed != indexed) {
- this.indexed = true; // once indexed, always index
- }
+ this.indexed |= indexed; // once indexed, always indexed
if (indexed) { // if updated field data is not for indexing, leave the updates out
- if (this.storeTermVector != storeTermVector) {
- this.storeTermVector = true; // once vector, always vector
- }
- if (this.storePayloads != storePayloads) {
- this.storePayloads = true;
- }
+ this.storeTermVector |= storeTermVector; // once vector, always vector
+ this.storePayloads |= storePayloads;
if (this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
this.normType = null;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java Tue Apr 29 17:18:41 2014
@@ -17,6 +17,10 @@
package org.apache.lucene.index;
import org.apache.lucene.analysis.TokenStream; // javadocs
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.util.AttributeSource;
/**
@@ -37,6 +41,11 @@ public final class FieldInvertState {
float boost;
AttributeSource attributeSource;
+ OffsetAttribute offsetAttribute;
+ PositionIncrementAttribute posIncrAttribute;
+ PayloadAttribute payloadAttribute;
+ TermToBytesRefAttribute termAttribute;
+
/** Creates {code FieldInvertState} for the specified
* field name. */
public FieldInvertState(String name) {
@@ -65,7 +74,20 @@ public final class FieldInvertState {
maxTermFrequency = 0;
uniqueTermCount = 0;
boost = 1.0f;
- attributeSource = null;
+ }
+
+ // TODO: better name?
+ /**
+ * Sets attributeSource to a new instance.
+ */
+ void setAttributeSource(AttributeSource attributeSource) {
+ if (this.attributeSource != attributeSource) {
+ this.attributeSource = attributeSource;
+ termAttribute = attributeSource.getAttribute(TermToBytesRefAttribute.class);
+ posIncrAttribute = attributeSource.addAttribute(PositionIncrementAttribute.class);
+ offsetAttribute = attributeSource.addAttribute(OffsetAttribute.class);
+ payloadAttribute = attributeSource.getAttribute(PayloadAttribute.class);
+ }
}
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java Tue Apr 29 17:18:41 2014
@@ -126,7 +126,7 @@ class FreqProxFields extends Fields {
@Override
public boolean hasPayloads() {
- return terms.hasPayloads;
+ return terms.sawPayloads;
}
}
@@ -140,10 +140,10 @@ class FreqProxFields extends Fields {
public FreqProxTermsEnum(FreqProxTermsWriterPerField terms) {
this.terms = terms;
- this.numTerms = terms.termsHashPerField.bytesHash.size();
+ this.numTerms = terms.bytesHash.size();
sortedTermIDs = terms.sortedTermIDs;
assert sortedTermIDs != null;
- postingsArray = (FreqProxPostingsArray) terms.termsHashPerField.postingsArray;
+ postingsArray = (FreqProxPostingsArray) terms.postingsArray;
}
public void reset() {
@@ -161,7 +161,7 @@ class FreqProxFields extends Fields {
while (hi >= lo) {
int mid = (lo + hi) >>> 1;
int textStart = postingsArray.textStarts[sortedTermIDs[mid]];
- terms.termsHashPerField.bytePool.setBytesRef(scratch, textStart);
+ terms.bytePool.setBytesRef(scratch, textStart);
int cmp = scratch.compareTo(text);
if (cmp < 0) {
lo = mid + 1;
@@ -186,7 +186,7 @@ class FreqProxFields extends Fields {
public void seekExact(long ord) {
this.ord = (int) ord;
int textStart = postingsArray.textStarts[sortedTermIDs[this.ord]];
- terms.termsHashPerField.bytePool.setBytesRef(scratch, textStart);
+ terms.bytePool.setBytesRef(scratch, textStart);
}
@Override
@@ -196,7 +196,7 @@ class FreqProxFields extends Fields {
return null;
} else {
int textStart = postingsArray.textStarts[sortedTermIDs[ord]];
- terms.termsHashPerField.bytePool.setBytesRef(scratch, textStart);
+ terms.bytePool.setBytesRef(scratch, textStart);
return scratch;
}
}
@@ -324,7 +324,7 @@ class FreqProxFields extends Fields {
public void reset(int termID) {
this.termID = termID;
- terms.termsHashPerField.initReader(reader, termID, 0);
+ terms.initReader(reader, termID, 0);
ended = false;
docID = 0;
}
@@ -415,8 +415,8 @@ class FreqProxFields extends Fields {
public void reset(int termID) {
this.termID = termID;
- terms.termsHashPerField.initReader(reader, termID, 0);
- terms.termsHashPerField.initReader(posReader, termID, 1);
+ terms.initReader(reader, termID, 0);
+ terms.initReader(posReader, termID, 1);
ended = false;
docID = 0;
posLeft = 0;
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Tue Apr 29 17:18:41 2014
@@ -23,13 +23,13 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
-final class FreqProxTermsWriter extends TermsHashConsumer {
+final class FreqProxTermsWriter extends TermsHash {
- @Override
- void abort() {}
+ public FreqProxTermsWriter(DocumentsWriterPerThread docWriter, TermsHash termVectors) {
+ super(docWriter, true, termVectors);
+ }
private void applyDeletes(SegmentWriteState state, Fields fields) throws IOException {
// Process any pending Term deletes for this newly
@@ -77,21 +77,16 @@ final class FreqProxTermsWriter extends
}
}
- // TODO: would be nice to factor out more of this, eg the
- // FreqProxFieldMergeState, and code to visit all Fields
- // under the same FieldInfo together, up into TermsHash*.
- // Other writers would presumably share alot of this...
-
@Override
- public void flush(Map<String,TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ public void flush(Map<String,TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ super.flush(fieldsToFlush, state);
- // Gather all FieldData's that have postings, across all
- // ThreadStates
+ // Gather all fields that saw any postings:
List<FreqProxTermsWriterPerField> allFields = new ArrayList<>();
- for (TermsHashConsumerPerField f : fieldsToFlush.values()) {
+ for (TermsHashPerField f : fieldsToFlush.values()) {
final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f;
- if (perField.termsHashPerField.bytesHash.size() > 0) {
+ if (perField.bytesHash.size() > 0) {
perField.sortPostings();
assert perField.fieldInfo.isIndexed();
allFields.add(perField);
@@ -108,18 +103,8 @@ final class FreqProxTermsWriter extends
state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state).write(fields);
}
- BytesRef payload;
-
- @Override
- public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo) {
- return new FreqProxTermsWriterPerField(termsHashPerField, this, fieldInfo);
- }
-
- @Override
- void finishDocument(TermsHash termsHash) {
- }
-
@Override
- void startDocument() {
+ public TermsHashPerField addField(FieldInvertState invertState, FieldInfo fieldInfo) {
+ return new FreqProxTermsWriterPerField(invertState, this, fieldInfo, nextTermsHash.addField(invertState, fieldInfo));
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Tue Apr 29 17:18:41 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -26,16 +28,13 @@ import org.apache.lucene.util.RamUsageEs
// TODO: break into separate freq and prox writers as
// codecs; make separate container (tii/tis/skip/*) that can
// be configured as any number of files 1..N
-final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implements Comparable<FreqProxTermsWriterPerField> {
+final class FreqProxTermsWriterPerField extends TermsHashPerField {
+
+ private FreqProxPostingsArray freqProxPostingsArray;
- final FreqProxTermsWriter parent;
- final TermsHashPerField termsHashPerField;
- final FieldInfo fieldInfo;
- final DocumentsWriterPerThread.DocState docState;
- final FieldInvertState fieldState;
- boolean hasFreq;
- boolean hasProx;
- boolean hasOffsets;
+ final boolean hasFreq;
+ final boolean hasProx;
+ final boolean hasOffsets;
PayloadAttribute payloadAttribute;
OffsetAttribute offsetAttribute;
long sumTotalTermFreq;
@@ -44,111 +43,67 @@ final class FreqProxTermsWriterPerField
// How many docs have this field:
int docCount;
- public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriter parent, FieldInfo fieldInfo) {
- this.termsHashPerField = termsHashPerField;
- this.parent = parent;
- this.fieldInfo = fieldInfo;
- docState = termsHashPerField.docState;
- fieldState = termsHashPerField.fieldState;
- setIndexOptions(fieldInfo.getIndexOptions());
- }
-
- @Override
- int getStreamCount() {
- if (!hasProx) {
- return 1;
- } else {
- return 2;
- }
+ /** Set to true if any token had a payload in the current
+ * segment. */
+ boolean sawPayloads;
+
+ public FreqProxTermsWriterPerField(FieldInvertState invertState, TermsHash termsHash, FieldInfo fieldInfo, TermsHashPerField nextPerField) {
+ super(fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 ? 2 : 1, invertState, termsHash, nextPerField, fieldInfo);
+ IndexOptions indexOptions = fieldInfo.getIndexOptions();
+ assert indexOptions != null;
+ hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
}
@Override
- void finish() {
+ void finish() throws IOException {
+ super.finish();
sumDocFreq += fieldState.uniqueTermCount;
sumTotalTermFreq += fieldState.length;
if (fieldState.length > 0) {
docCount++;
}
- if (hasPayloads) {
+ if (sawPayloads) {
fieldInfo.setStorePayloads();
}
}
- boolean hasPayloads;
-
- @Override
- void skippingLongTerm() {}
-
@Override
- public int compareTo(FreqProxTermsWriterPerField other) {
- return fieldInfo.name.compareTo(other.fieldInfo.name);
- }
-
- private void setIndexOptions(IndexOptions indexOptions) {
- if (indexOptions == null) {
- // field could later be updated with indexed=true, so set everything on
- hasFreq = hasProx = hasOffsets = true;
- } else {
- hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
- }
-
- @Override
- boolean start(IndexableField[] fields, int count) {
+ boolean start(IndexableField f, boolean first) {
+ super.start(f, first);
+ payloadAttribute = fieldState.payloadAttribute;
+ offsetAttribute = fieldState.offsetAttribute;
return true;
}
- @Override
- void start(IndexableField f) {
- if (fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
- payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
- } else {
- payloadAttribute = null;
- }
- if (hasOffsets) {
- offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
- } else {
- offsetAttribute = null;
- }
- }
-
- void writeProx(final int termID, int proxCode) {
- //System.out.println("writeProx termID=" + termID + " proxCode=" + proxCode);
- assert hasProx;
- final BytesRef payload;
+ void writeProx(int termID, int proxCode) {
if (payloadAttribute == null) {
- payload = null;
- } else {
- payload = payloadAttribute.getPayload();
- }
-
- if (payload != null && payload.length > 0) {
- termsHashPerField.writeVInt(1, (proxCode<<1)|1);
- termsHashPerField.writeVInt(1, payload.length);
- termsHashPerField.writeBytes(1, payload.bytes, payload.offset, payload.length);
- hasPayloads = true;
+ writeVInt(1, proxCode<<1);
} else {
- termsHashPerField.writeVInt(1, proxCode<<1);
+ BytesRef payload = payloadAttribute.getPayload();
+ if (payload != null && payload.length > 0) {
+ writeVInt(1, (proxCode<<1)|1);
+ writeVInt(1, payload.length);
+ writeBytes(1, payload.bytes, payload.offset, payload.length);
+ sawPayloads = true;
+ } else {
+ writeVInt(1, proxCode<<1);
+ }
}
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
- postings.lastPositions[termID] = fieldState.position;
+ assert postingsArray == freqProxPostingsArray;
+ freqProxPostingsArray.lastPositions[termID] = fieldState.position;
}
- void writeOffsets(final int termID, int offsetAccum) {
- assert hasOffsets;
+ void writeOffsets(int termID, int offsetAccum) {
final int startOffset = offsetAccum + offsetAttribute.startOffset();
final int endOffset = offsetAccum + offsetAttribute.endOffset();
- //System.out.println("writeOffsets termID=" + termID + " prevOffset=" + prevOffset + " startOff=" + startOffset + " endOff=" + endOffset);
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
- assert startOffset - postings.lastOffsets[termID] >= 0;
- termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
-
- postings.lastOffsets[termID] = startOffset;
+ assert startOffset - freqProxPostingsArray.lastOffsets[termID] >= 0;
+ writeVInt(1, startOffset - freqProxPostingsArray.lastOffsets[termID]);
+ writeVInt(1, endOffset - startOffset);
+ freqProxPostingsArray.lastOffsets[termID] = startOffset;
}
@Override
@@ -157,9 +112,11 @@ final class FreqProxTermsWriterPerField
// flush
assert docState.testPoint("FreqProxTermsWriterPerField.newTerm start");
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
+ final FreqProxPostingsArray postings = freqProxPostingsArray;
+
postings.lastDocIDs[termID] = docState.docID;
if (!hasFreq) {
+ assert postings.termFreqs == null;
postings.lastDocCodes[termID] = docState.docID;
} else {
postings.lastDocCodes[termID] = docState.docID << 1;
@@ -182,15 +139,16 @@ final class FreqProxTermsWriterPerField
assert docState.testPoint("FreqProxTermsWriterPerField.addTerm start");
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
+ final FreqProxPostingsArray postings = freqProxPostingsArray;
assert !hasFreq || postings.termFreqs[termID] > 0;
if (!hasFreq) {
assert postings.termFreqs == null;
if (docState.docID != postings.lastDocIDs[termID]) {
+ // New document; now encode docCode for previous doc:
assert docState.docID > postings.lastDocIDs[termID];
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
+ writeVInt(0, postings.lastDocCodes[termID]);
postings.lastDocCodes[termID] = docState.docID - postings.lastDocIDs[termID];
postings.lastDocIDs[termID] = docState.docID;
fieldState.uniqueTermCount++;
@@ -203,11 +161,13 @@ final class FreqProxTermsWriterPerField
// Now that we know doc freq for previous doc,
// write it & lastDocCode
if (1 == postings.termFreqs[termID]) {
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]|1);
+ writeVInt(0, postings.lastDocCodes[termID]|1);
} else {
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
- termsHashPerField.writeVInt(0, postings.termFreqs[termID]);
+ writeVInt(0, postings.lastDocCodes[termID]);
+ writeVInt(0, postings.termFreqs[termID]);
}
+
+ // Init freq for the current document
postings.termFreqs[termID] = 1;
fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1;
@@ -226,15 +186,25 @@ final class FreqProxTermsWriterPerField
fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.termFreqs[termID]);
if (hasProx) {
writeProx(termID, fieldState.position-postings.lastPositions[termID]);
- }
- if (hasOffsets) {
- writeOffsets(termID, fieldState.offset);
+ if (hasOffsets) {
+ writeOffsets(termID, fieldState.offset);
+ }
}
}
}
@Override
+ public void newPostingsArray() {
+ freqProxPostingsArray = (FreqProxPostingsArray) postingsArray;
+ }
+
+ @Override
ParallelPostingsArray createPostingsArray(int size) {
+ IndexOptions indexOptions = fieldInfo.getIndexOptions();
+ assert indexOptions != null;
+ boolean hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ boolean hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ boolean hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
return new FreqProxPostingsArray(size, hasFreq, hasProx, hasOffsets);
}
@@ -307,15 +277,4 @@ final class FreqProxTermsWriterPerField
return bytes;
}
}
-
- public void abort() {}
-
- BytesRef payload;
-
- int[] sortedTermIDs;
-
- void sortPostings() {
- assert sortedTermIDs == null;
- sortedTermIDs = termsHashPerField.sortPostings();
- }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Tue Apr 29 17:18:41 2014
@@ -39,6 +39,8 @@ public interface IndexableFieldType {
* <p>
* This has no effect if {@link #indexed()} returns false.
*/
+ // TODO: shouldn't we remove this? Whether/how a field is
+ // tokenized is an impl detail under Field?
public boolean tokenized();
/**
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java Tue Apr 29 17:18:41 2014
@@ -97,10 +97,6 @@ class NumericDocValuesWriter extends Doc
});
}
- @Override
- public void abort() {
- }
-
// iterates over the values we have in ram
private class NumericIterator implements Iterator<Number> {
final AppendingDeltaPackedLongBuffer.Iterator iter = pending.iterator();
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/PrefixCodedTerms.java Tue Apr 29 17:18:41 2014
@@ -97,7 +97,7 @@ class PrefixCodedTerms implements Iterab
/** Builds a PrefixCodedTerms: call add repeatedly, then finish. */
public static class Builder {
private RAMFile buffer = new RAMFile();
- private RAMOutputStream output = new RAMOutputStream(buffer);
+ private RAMOutputStream output = new RAMOutputStream(buffer, false);
private Term lastTerm = new Term("");
/** add a term */
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java Tue Apr 29 17:18:41 2014
@@ -139,10 +139,6 @@ class SortedDocValuesWriter extends DocV
});
}
- @Override
- public void abort() {
- }
-
// iterates over the unique values we have in ram
private class ValuesIterator implements Iterator<BytesRef> {
final int sortedValues[];
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java Tue Apr 29 17:18:41 2014
@@ -187,10 +187,6 @@ class SortedSetDocValuesWriter extends D
});
}
- @Override
- public void abort() {
- }
-
// iterates over the unique values we have in ram
private class ValuesIterator implements Iterator<BytesRef> {
final int sortedValues[];
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java Tue Apr 29 17:18:41 2014
@@ -29,28 +29,32 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
-final class TermVectorsConsumer extends TermsHashConsumer {
+final class TermVectorsConsumer extends TermsHash {
TermVectorsWriter writer;
- final DocumentsWriterPerThread docWriter;
- final DocumentsWriterPerThread.DocState docState;
+
+ /** Scratch term used by TermVectorsConsumerPerField.finishDocument. */
final BytesRef flushTerm = new BytesRef();
- // Used by perField when serializing the term vectors
+ final DocumentsWriterPerThread docWriter;
+
+ /** Used by TermVectorsConsumerPerField when serializing
+ * the term vectors. */
final ByteSliceReader vectorSliceReaderPos = new ByteSliceReader();
final ByteSliceReader vectorSliceReaderOff = new ByteSliceReader();
+
boolean hasVectors;
int numVectorFields;
int lastDocID;
private TermVectorsConsumerPerField[] perFields = new TermVectorsConsumerPerField[1];
public TermVectorsConsumer(DocumentsWriterPerThread docWriter) {
+ super(docWriter, false, null);
this.docWriter = docWriter;
- docState = docWriter.docState;
}
@Override
- void flush(Map<String, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ void flush(Map<String, TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
if (writer != null) {
int numDocs = state.segmentInfo.getDocCount();
assert numDocs > 0;
@@ -78,7 +82,7 @@ final class TermVectorsConsumer extends
}
}
- private final void initTermVectorsWriter() throws IOException {
+ private void initTermVectorsWriter() throws IOException {
if (writer == null) {
IOContext context = new IOContext(new FlushInfo(docWriter.getNumDocsInRAM(), docWriter.bytesUsed()));
writer = docWriter.codec.termVectorsFormat().vectorsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
@@ -87,7 +91,7 @@ final class TermVectorsConsumer extends
}
@Override
- void finishDocument(TermsHash termsHash) throws IOException {
+ void finishDocument() throws IOException {
assert docWriter.testPoint("TermVectorsTermsWriter.finishDocument start");
@@ -95,6 +99,9 @@ final class TermVectorsConsumer extends
return;
}
+ // Fields in term vectors are UTF16 sorted:
+ ArrayUtil.introSort(perFields, 0, numVectorFields);
+
initTermVectorsWriter();
fill(docState.docID);
@@ -110,32 +117,35 @@ final class TermVectorsConsumer extends
lastDocID++;
- termsHash.reset();
- reset();
+ super.reset();
+ resetFields();
assert docWriter.testPoint("TermVectorsTermsWriter.finishDocument end");
}
@Override
public void abort() {
hasVectors = false;
+ try {
+ super.abort();
+ } finally {
+ if (writer != null) {
+ writer.abort();
+ writer = null;
+ }
- if (writer != null) {
- writer.abort();
- writer = null;
+ lastDocID = 0;
+ reset();
}
-
- lastDocID = 0;
- reset();
}
- void reset() {
- Arrays.fill(perFields, null);// don't hang onto stuff from previous doc
+ void resetFields() {
+ Arrays.fill(perFields, null); // don't hang onto stuff from previous doc
numVectorFields = 0;
}
@Override
- public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo) {
- return new TermVectorsConsumerPerField(termsHashPerField, this, fieldInfo);
+ public TermsHashPerField addField(FieldInvertState invertState, FieldInfo fieldInfo) {
+ return new TermVectorsConsumerPerField(invertState, this, fieldInfo);
}
void addFieldToFlush(TermVectorsConsumerPerField fieldToFlush) {
@@ -151,24 +161,7 @@ final class TermVectorsConsumer extends
@Override
void startDocument() {
- assert clearLastVectorFieldName();
- reset();
- }
-
- // Called only by assert
- final boolean clearLastVectorFieldName() {
- lastVectorFieldName = null;
- return true;
- }
-
- // Called only by assert
- String lastVectorFieldName;
- final boolean vectorFieldsInOrder(FieldInfo fi) {
- try {
- return lastVectorFieldName != null ? lastVectorFieldName.compareTo(fi.name) < 0 : true;
- } finally {
- lastVectorFieldName = fi.name;
- }
+ resetFields();
+ numVectorFields = 0;
}
-
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java Tue Apr 29 17:18:41 2014
@@ -22,149 +22,69 @@ import java.io.IOException;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.codecs.TermVectorsWriter;
-import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
-final class TermVectorsConsumerPerField extends TermsHashConsumerPerField {
+final class TermVectorsConsumerPerField extends TermsHashPerField {
+
+ private TermVectorsPostingsArray termVectorsPostingsArray;
- final TermsHashPerField termsHashPerField;
final TermVectorsConsumer termsWriter;
- final FieldInfo fieldInfo;
- final DocumentsWriterPerThread.DocState docState;
- final FieldInvertState fieldState;
boolean doVectors;
boolean doVectorPositions;
boolean doVectorOffsets;
boolean doVectorPayloads;
- int maxNumPostings;
OffsetAttribute offsetAttribute;
PayloadAttribute payloadAttribute;
boolean hasPayloads; // if enabled, and we actually saw any for this field
- public TermVectorsConsumerPerField(TermsHashPerField termsHashPerField, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
- this.termsHashPerField = termsHashPerField;
+ public TermVectorsConsumerPerField(FieldInvertState invertState, TermVectorsConsumer termsWriter, FieldInfo fieldInfo) {
+ super(2, invertState, termsWriter, null, fieldInfo);
this.termsWriter = termsWriter;
- this.fieldInfo = fieldInfo;
- docState = termsHashPerField.docState;
- fieldState = termsHashPerField.fieldState;
- }
-
- @Override
- int getStreamCount() {
- return 2;
}
- @Override
- boolean start(IndexableField[] fields, int count) {
- doVectors = false;
- doVectorPositions = false;
- doVectorOffsets = false;
- doVectorPayloads = false;
- hasPayloads = false;
-
- for(int i=0;i<count;i++) {
- IndexableField field = fields[i];
- if (field.fieldType().indexed()) {
- if (field.fieldType().storeTermVectors()) {
- doVectors = true;
- doVectorPositions |= field.fieldType().storeTermVectorPositions();
- doVectorOffsets |= field.fieldType().storeTermVectorOffsets();
- if (doVectorPositions) {
- doVectorPayloads |= field.fieldType().storeTermVectorPayloads();
- } else if (field.fieldType().storeTermVectorPayloads()) {
- // TODO: move this check somewhere else, and impl the other missing ones
- throw new IllegalArgumentException("cannot index term vector payloads without term vector positions (field=\"" + field.name() + "\")");
- }
- } else {
- if (field.fieldType().storeTermVectorOffsets()) {
- throw new IllegalArgumentException("cannot index term vector offsets when term vectors are not indexed (field=\"" + field.name() + "\")");
- }
- if (field.fieldType().storeTermVectorPositions()) {
- throw new IllegalArgumentException("cannot index term vector positions when term vectors are not indexed (field=\"" + field.name() + "\")");
- }
- if (field.fieldType().storeTermVectorPayloads()) {
- throw new IllegalArgumentException("cannot index term vector payloads when term vectors are not indexed (field=\"" + field.name() + "\")");
- }
- }
- } else {
- if (field.fieldType().storeTermVectors()) {
- throw new IllegalArgumentException("cannot index term vectors when field is not indexed (field=\"" + field.name() + "\")");
- }
- if (field.fieldType().storeTermVectorOffsets()) {
- throw new IllegalArgumentException("cannot index term vector offsets when field is not indexed (field=\"" + field.name() + "\")");
- }
- if (field.fieldType().storeTermVectorPositions()) {
- throw new IllegalArgumentException("cannot index term vector positions when field is not indexed (field=\"" + field.name() + "\")");
- }
- if (field.fieldType().storeTermVectorPayloads()) {
- throw new IllegalArgumentException("cannot index term vector payloads when field is not indexed (field=\"" + field.name() + "\")");
- }
- }
- }
-
- if (doVectors) {
- termsWriter.hasVectors = true;
- if (termsHashPerField.bytesHash.size() != 0) {
- // Only necessary if previous doc hit a
- // non-aborting exception while writing vectors in
- // this field:
- termsHashPerField.reset();
- }
- }
-
- // TODO: only if needed for performance
- //perThread.postingsCount = 0;
-
- return doVectors;
- }
-
- public void abort() {}
-
/** Called once per field per document if term vectors
* are enabled, to write the vectors to
* RAMOutputStream, which is then quickly flushed to
* the real term vectors files in the Directory. */ @Override
void finish() {
- if (!doVectors || termsHashPerField.bytesHash.size() == 0) {
+ if (!doVectors || bytesHash.size() == 0) {
return;
}
-
termsWriter.addFieldToFlush(this);
}
void finishDocument() throws IOException {
+ if (doVectors == false) {
+ return;
+ }
+
+ doVectors = false;
+
assert docState.testPoint("TermVectorsTermsWriterPerField.finish start");
- final int numPostings = termsHashPerField.bytesHash.size();
+ final int numPostings = bytesHash.size();
final BytesRef flushTerm = termsWriter.flushTerm;
assert numPostings >= 0;
- if (numPostings > maxNumPostings)
- maxNumPostings = numPostings;
-
// This is called once, after inverting all occurrences
// of a given field in the doc. At this point we flush
// our hash into the DocWriter.
- assert termsWriter.vectorFieldsInOrder(fieldInfo);
-
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
+ TermVectorsPostingsArray postings = termVectorsPostingsArray;
final TermVectorsWriter tv = termsWriter.writer;
- final int[] termIDs = termsHashPerField.sortPostings();
+ final int[] termIDs = sortPostings();
tv.startField(fieldInfo, numPostings, doVectorPositions, doVectorOffsets, hasPayloads);
final ByteSliceReader posReader = doVectorPositions ? termsWriter.vectorSliceReaderPos : null;
final ByteSliceReader offReader = doVectorOffsets ? termsWriter.vectorSliceReaderOff : null;
- final ByteBlockPool termBytePool = termsHashPerField.termBytePool;
-
for(int j=0;j<numPostings;j++) {
final int termID = termIDs[j];
final int freq = postings.freqs[termID];
@@ -175,10 +95,10 @@ final class TermVectorsConsumerPerField
if (doVectorPositions || doVectorOffsets) {
if (posReader != null) {
- termsHashPerField.initReader(posReader, termID, 0);
+ initReader(posReader, termID, 0);
}
if (offReader != null) {
- termsHashPerField.initReader(offReader, termID, 1);
+ initReader(offReader, termID, 1);
}
tv.addProx(freq, posReader, offReader);
}
@@ -186,23 +106,91 @@ final class TermVectorsConsumerPerField
}
tv.finishField();
- termsHashPerField.reset();
+ reset();
fieldInfo.setStoreTermVectors();
}
@Override
- void start(IndexableField f) {
- if (doVectorOffsets) {
- offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
+ boolean start(IndexableField field, boolean first) {
+ assert field.fieldType().indexed();
+
+ if (first) {
+
+ if (bytesHash.size() != 0) {
+ // Only necessary if previous doc hit a
+ // non-aborting exception while writing vectors in
+ // this field:
+ reset();
+ }
+
+ bytesHash.reinit();
+
+ hasPayloads = false;
+
+ doVectors = field.fieldType().storeTermVectors();
+
+ if (doVectors) {
+
+ termsWriter.hasVectors = true;
+
+ doVectorPositions = field.fieldType().storeTermVectorPositions();
+
+ // Somewhat confusingly, unlike postings, you are
+ // allowed to index TV offsets without TV positions:
+ doVectorOffsets = field.fieldType().storeTermVectorOffsets();
+
+ if (doVectorPositions) {
+ doVectorPayloads = field.fieldType().storeTermVectorPayloads();
+ } else {
+ doVectorPayloads = false;
+ if (field.fieldType().storeTermVectorPayloads()) {
+ // TODO: move this check somewhere else, and impl the other missing ones
+ throw new IllegalArgumentException("cannot index term vector payloads without term vector positions (field=\"" + field.name() + "\")");
+ }
+ }
+
+ } else {
+ if (field.fieldType().storeTermVectorOffsets()) {
+ throw new IllegalArgumentException("cannot index term vector offsets when term vectors are not indexed (field=\"" + field.name() + "\")");
+ }
+ if (field.fieldType().storeTermVectorPositions()) {
+ throw new IllegalArgumentException("cannot index term vector positions when term vectors are not indexed (field=\"" + field.name() + "\")");
+ }
+ if (field.fieldType().storeTermVectorPayloads()) {
+ throw new IllegalArgumentException("cannot index term vector payloads when term vectors are not indexed (field=\"" + field.name() + "\")");
+ }
+ }
} else {
- offsetAttribute = null;
+ if (doVectors != field.fieldType().storeTermVectors()) {
+ throw new IllegalArgumentException("all instances of a given field name must have the same term vectors settings (storeTermVectors changed for field=\"" + field.name() + "\")");
+ }
+ if (doVectorPositions != field.fieldType().storeTermVectorPositions()) {
+ throw new IllegalArgumentException("all instances of a given field name must have the same term vectors settings (storeTermVectorPositions changed for field=\"" + field.name() + "\")");
+ }
+ if (doVectorOffsets != field.fieldType().storeTermVectorOffsets()) {
+ throw new IllegalArgumentException("all instances of a given field name must have the same term vectors settings (storeTermVectorOffsets changed for field=\"" + field.name() + "\")");
+ }
+ if (doVectorPayloads != field.fieldType().storeTermVectorPayloads()) {
+ throw new IllegalArgumentException("all instances of a given field name must have the same term vectors settings (storeTermVectorPayloads changed for field=\"" + field.name() + "\")");
+ }
}
- if (doVectorPayloads && fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
- payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
- } else {
- payloadAttribute = null;
+
+ if (doVectors) {
+ if (doVectorOffsets) {
+ offsetAttribute = fieldState.offsetAttribute;
+ assert offsetAttribute != null;
+ }
+
+ if (doVectorPayloads) {
+ // Can be null:
+ payloadAttribute = fieldState.payloadAttribute;
+ } else {
+ payloadAttribute = null;
+ }
}
+
+ return doVectors;
}
void writeProx(TermVectorsPostingsArray postings, int termID) {
@@ -210,8 +198,8 @@ final class TermVectorsConsumerPerField
int startOffset = fieldState.offset + offsetAttribute.startOffset();
int endOffset = fieldState.offset + offsetAttribute.endOffset();
- termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
+ writeVInt(1, startOffset - postings.lastOffsets[termID]);
+ writeVInt(1, endOffset - startOffset);
postings.lastOffsets[termID] = endOffset;
}
@@ -225,12 +213,12 @@ final class TermVectorsConsumerPerField
final int pos = fieldState.position - postings.lastPositions[termID];
if (payload != null && payload.length > 0) {
- termsHashPerField.writeVInt(0, (pos<<1)|1);
- termsHashPerField.writeVInt(0, payload.length);
- termsHashPerField.writeBytes(0, payload.bytes, payload.offset, payload.length);
+ writeVInt(0, (pos<<1)|1);
+ writeVInt(0, payload.length);
+ writeBytes(0, payload.bytes, payload.offset, payload.length);
hasPayloads = true;
} else {
- termsHashPerField.writeVInt(0, pos<<1);
+ writeVInt(0, pos<<1);
}
postings.lastPositions[termID] = fieldState.position;
}
@@ -239,7 +227,7 @@ final class TermVectorsConsumerPerField
@Override
void newTerm(final int termID) {
assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
+ TermVectorsPostingsArray postings = termVectorsPostingsArray;
postings.freqs[termID] = 1;
postings.lastOffsets[termID] = 0;
@@ -251,7 +239,7 @@ final class TermVectorsConsumerPerField
@Override
void addTerm(final int termID) {
assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
+ TermVectorsPostingsArray postings = termVectorsPostingsArray;
postings.freqs[termID]++;
@@ -259,7 +247,9 @@ final class TermVectorsConsumerPerField
}
@Override
- void skippingLongTerm() {}
+ public void newPostingsArray() {
+ termVectorsPostingsArray = (TermVectorsPostingsArray) postingsArray;
+ }
@Override
ParallelPostingsArray createPostingsArray(int size) {
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHash.java?rev=1591025&r1=1591024&r2=1591025&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHash.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/index/TermsHash.java Tue Apr 29 17:18:41 2014
@@ -22,21 +22,17 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.util.ByteBlockPool;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
import org.apache.lucene.util.IntBlockPool;
-/** This class implements {@link InvertedDocConsumer}, which
- * is passed each token produced by the analyzer on each
- * field. It stores these tokens in a hash table, and
- * allocates separate byte streams per token. Consumers of
- * this class, eg {@link FreqProxTermsWriter} and {@link
- * TermVectorsConsumer}, write their own byte streams
- * under each term.
- */
-final class TermsHash extends InvertedDocConsumer {
+/** This class is passed each token produced by the analyzer
+ * on each field during indexing, and it stores these
+ * tokens in a hash table, and allocates separate byte
+ * streams per token. Consumers of this class, eg {@link
+ * FreqProxTermsWriter} and {@link TermVectorsConsumer},
+ * write their own byte streams under each term. */
+abstract class TermsHash {
- final TermsHashConsumer consumer;
final TermsHash nextTermsHash;
final IntBlockPool intPool;
@@ -44,21 +40,12 @@ final class TermsHash extends InvertedDo
ByteBlockPool termBytePool;
final Counter bytesUsed;
- final boolean primary;
final DocumentsWriterPerThread.DocState docState;
- // Used when comparing postings via termRefComp, in TermsHashPerField
- final BytesRef tr1 = new BytesRef();
- final BytesRef tr2 = new BytesRef();
-
- // Used by perField to obtain terms from the analysis chain
- final BytesRef termBytesRef = new BytesRef(10);
-
final boolean trackAllocations;
- public TermsHash(final DocumentsWriterPerThread docWriter, final TermsHashConsumer consumer, boolean trackAllocations, final TermsHash nextTermsHash) {
+ TermsHash(final DocumentsWriterPerThread docWriter, boolean trackAllocations, TermsHash nextTermsHash) {
this.docState = docWriter.docState;
- this.consumer = consumer;
this.trackAllocations = trackAllocations;
this.nextTermsHash = nextTermsHash;
this.bytesUsed = trackAllocations ? docWriter.bytesUsed : Counter.newCounter();
@@ -67,19 +54,14 @@ final class TermsHash extends InvertedDo
if (nextTermsHash != null) {
// We are primary
- primary = true;
termBytePool = bytePool;
nextTermsHash.termBytePool = bytePool;
- } else {
- primary = false;
}
}
- @Override
public void abort() {
- reset();
try {
- consumer.abort();
+ reset();
} finally {
if (nextTermsHash != null) {
nextTermsHash.abort();
@@ -94,50 +76,27 @@ final class TermsHash extends InvertedDo
bytePool.reset(false, false);
}
- @Override
- void flush(Map<String,InvertedDocConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
- Map<String,TermsHashConsumerPerField> childFields = new HashMap<>();
- Map<String,InvertedDocConsumerPerField> nextChildFields;
-
+ void flush(Map<String,TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
if (nextTermsHash != null) {
- nextChildFields = new HashMap<>();
- } else {
- nextChildFields = null;
- }
-
- for (final Map.Entry<String,InvertedDocConsumerPerField> entry : fieldsToFlush.entrySet()) {
- TermsHashPerField perField = (TermsHashPerField) entry.getValue();
- childFields.put(entry.getKey(), perField.consumer);
- if (nextTermsHash != null) {
- nextChildFields.put(entry.getKey(), perField.nextPerField);
+ Map<String,TermsHashPerField> nextChildFields = new HashMap<>();
+ for (final Map.Entry<String,TermsHashPerField> entry : fieldsToFlush.entrySet()) {
+ nextChildFields.put(entry.getKey(), entry.getValue().nextPerField);
}
- }
-
- consumer.flush(childFields, state);
-
- if (nextTermsHash != null) {
nextTermsHash.flush(nextChildFields, state);
}
}
- @Override
- InvertedDocConsumerPerField addField(DocInverterPerField docInverterPerField, final FieldInfo fieldInfo) {
- return new TermsHashPerField(docInverterPerField, this, nextTermsHash, fieldInfo);
- }
+ abstract TermsHashPerField addField(FieldInvertState fieldInvertState, final FieldInfo fieldInfo);
- @Override
void finishDocument() throws IOException {
- consumer.finishDocument(this);
if (nextTermsHash != null) {
- nextTermsHash.consumer.finishDocument(nextTermsHash);
+ nextTermsHash.finishDocument();
}
}
- @Override
void startDocument() throws IOException {
- consumer.startDocument();
if (nextTermsHash != null) {
- nextTermsHash.consumer.startDocument();
+ nextTermsHash.startDocument();
}
}
}