You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/04/28 19:31:14 UTC
svn commit: r1590721 [1/2] - in /lucene/dev/branches/lucene5611/lucene:
codecs/src/java/org/apache/lucene/codecs/simpletext/
core/src/java/org/apache/lucene/codecs/
core/src/java/org/apache/lucene/codecs/compressing/
core/src/java/org/apache/lucene/cod...
Author: mikemccand
Date: Mon Apr 28 17:31:13 2014
New Revision: 1590721
URL: http://svn.apache.org/r1590721
Log:
LUCENE-5611: put current patch on branch
Added:
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (with props)
Removed:
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocFieldConsumerPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessor.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocFieldProcessorPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocInverter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocInverterPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocValuesProcessor.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/InvertedDocConsumerPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/InvertedDocEndConsumerPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NormsConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NormsConsumerPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/StoredFieldsConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/StoredFieldsProcessor.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermsHashConsumerPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TwoStoredFieldsConsumers.java
Modified:
lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/Field.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/FieldType.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumerPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermsHash.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermsHashPerField.java
lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/util/BytesRefHash.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/BinaryTokenStream.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/TestParallelReaderEmptyIndex.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java
lucene/dev/branches/lucene5611/lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/branches/lucene5611/lucene/test-framework/src/java/org/apache/lucene/codecs/asserting/AssertingStoredFieldsFormat.java
lucene/dev/branches/lucene5611/lucene/test-framework/src/java/org/apache/lucene/util/LuceneTestCase.java
Modified: lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java (original)
+++ lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsReader.java Mon Apr 28 17:31:13 2014
@@ -98,13 +98,12 @@ public class SimpleTextStoredFieldsReade
@Override
public void visitDocument(int n, StoredFieldVisitor visitor) throws IOException {
in.seek(offsets[n]);
- readLine();
- assert StringHelper.startsWith(scratch, NUM);
- int numFields = parseIntAt(NUM.length);
- for (int i = 0; i < numFields; i++) {
+ while (true) {
readLine();
- assert StringHelper.startsWith(scratch, FIELD);
+ if (StringHelper.startsWith(scratch, FIELD) == false) {
+ break;
+ }
int fieldNumber = parseIntAt(FIELD.length);
FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);
readLine();
Modified: lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextStoredFieldsWriter.java Mon Apr 28 17:31:13 2014
@@ -53,7 +53,6 @@ public class SimpleTextStoredFieldsWrite
final static BytesRef END = new BytesRef("END");
final static BytesRef DOC = new BytesRef("doc ");
- final static BytesRef NUM = new BytesRef(" numfields ");
final static BytesRef FIELD = new BytesRef(" field ");
final static BytesRef NAME = new BytesRef(" name ");
final static BytesRef TYPE = new BytesRef(" type ");
@@ -76,15 +75,11 @@ public class SimpleTextStoredFieldsWrite
}
@Override
- public void startDocument(int numStoredFields) throws IOException {
+ public void startDocument() throws IOException {
write(DOC);
write(Integer.toString(numDocsWritten));
newLine();
- write(NUM);
- write(Integer.toString(numStoredFields));
- newLine();
-
numDocsWritten++;
}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/StoredFieldsWriter.java Mon Apr 28 17:31:13 2014
@@ -31,7 +31,7 @@ import org.apache.lucene.index.AtomicRea
* Codec API for writing stored fields:
* <p>
* <ol>
- * <li>For every document, {@link #startDocument(int)} is called,
+ * <li>For every document, {@link #startDocument()} is called,
* informing the Codec how many fields will be written.
* <li>{@link #writeField(FieldInfo, StorableField)} is called for
* each field in the document.
@@ -51,10 +51,9 @@ public abstract class StoredFieldsWriter
/** Called before writing the stored fields of the document.
* {@link #writeField(FieldInfo, StorableField)} will be called
- * <code>numStoredFields</code> times. Note that this is
- * called even if the document has no stored fields, in
- * this case <code>numStoredFields</code> will be zero. */
- public abstract void startDocument(int numStoredFields) throws IOException;
+ * for each stored field. Note that this is
+ * called even if the document has no stored fields. */
+ public abstract void startDocument() throws IOException;
/** Called when a document and all its fields have been added. */
public void finishDocument() throws IOException {}
@@ -69,14 +68,14 @@ public abstract class StoredFieldsWriter
/** Called before {@link #close()}, passing in the number
* of documents that were written. Note that this is
* intentionally redundant (equivalent to the number of
- * calls to {@link #startDocument(int)}, but a Codec should
+ * calls to {@link #startDocument()}, but a Codec should
* check that this is the case to detect the JRE bug described
* in LUCENE-1282. */
public abstract void finish(FieldInfos fis, int numDocs) throws IOException;
/** Merges in the stored fields from the readers in
* <code>mergeState</code>. The default implementation skips
- * over deleted documents, and uses {@link #startDocument(int)},
+ * over deleted documents, and uses {@link #startDocument()},
* {@link #writeField(FieldInfo, StorableField)}, and {@link #finish(FieldInfos, int)},
* returning the number of documents that were written.
* Implementations can override this method for more sophisticated
@@ -109,12 +108,7 @@ public abstract class StoredFieldsWriter
/** sugar method for startDocument() + writeField() for every stored field in the document */
protected final void addDocument(Iterable<? extends StorableField> doc, FieldInfos fieldInfos) throws IOException {
- int storedCount = 0;
- for (StorableField field : doc) {
- storedCount++;
- }
-
- startDocument(storedCount);
+ startDocument();
for (StorableField field : doc) {
writeField(fieldInfos.fieldInfo(field.name()), field);
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingStoredFieldsWriter.java Mon Apr 28 17:31:13 2014
@@ -145,20 +145,23 @@ public final class CompressingStoredFiel
}
}
+ private int numStoredFieldsInDoc;
+
+ @Override
+ public void startDocument() throws IOException {
+ }
+
@Override
- public void startDocument(int numStoredFields) throws IOException {
+ public void finishDocument() throws IOException {
if (numBufferedDocs == this.numStoredFields.length) {
final int newLength = ArrayUtil.oversize(numBufferedDocs + 1, 4);
this.numStoredFields = Arrays.copyOf(this.numStoredFields, newLength);
endOffsets = Arrays.copyOf(endOffsets, newLength);
}
- this.numStoredFields[numBufferedDocs] = numStoredFields;
+ this.numStoredFields[numBufferedDocs] = numStoredFieldsInDoc;
+ numStoredFieldsInDoc = 0;
+ endOffsets[numBufferedDocs] = bufferedDocs.length;
++numBufferedDocs;
- }
-
- @Override
- public void finishDocument() throws IOException {
- endOffsets[numBufferedDocs - 1] = bufferedDocs.length;
if (triggerFlush()) {
flush();
}
@@ -242,6 +245,9 @@ public final class CompressingStoredFiel
@Override
public void writeField(FieldInfo info, StorableField field)
throws IOException {
+
+ ++numStoredFieldsInDoc;
+
int bits = 0;
final BytesRef bytes;
final String string;
@@ -391,8 +397,9 @@ public final class CompressingStoredFiel
// copy non-deleted docs
for (; docID < it.docBase + it.chunkDocs; docID = nextLiveDoc(docID + 1, liveDocs, maxDoc)) {
final int diff = docID - it.docBase;
- startDocument(it.numStoredFields[diff]);
+ startDocument();
bufferedDocs.writeBytes(it.bytes.bytes, it.bytes.offset + startOffsets[diff], it.lengths[diff]);
+ numStoredFieldsInDoc = it.numStoredFields[diff];
finishDocument();
++docCount;
mergeState.checkAbort.work(300);
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java Mon Apr 28 17:31:13 2014
@@ -36,6 +36,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -85,6 +86,7 @@ public final class Lucene40StoredFieldsW
private final String segment;
private IndexOutput fieldsStream;
private IndexOutput indexStream;
+ private final RAMOutputStream fieldsBuffer = new RAMOutputStream();
/** Sole constructor. */
public Lucene40StoredFieldsWriter(Directory directory, String segment, IOContext context) throws IOException {
@@ -109,14 +111,23 @@ public final class Lucene40StoredFieldsW
}
}
+ int numStoredFields;
+
// Writes the contents of buffer into the fields stream
// and adds a new entry for this document into the index
// stream. This assumes the buffer was already written
// in the correct fields format.
@Override
- public void startDocument(int numStoredFields) throws IOException {
+ public void startDocument() throws IOException {
indexStream.writeLong(fieldsStream.getFilePointer());
+ }
+
+ @Override
+ public void finishDocument() throws IOException {
fieldsStream.writeVInt(numStoredFields);
+ fieldsBuffer.writeTo(fieldsStream);
+ fieldsBuffer.reset();
+ numStoredFields = 0;
}
@Override
@@ -140,7 +151,9 @@ public final class Lucene40StoredFieldsW
@Override
public void writeField(FieldInfo info, StorableField field) throws IOException {
- fieldsStream.writeVInt(info.number);
+ numStoredFields++;
+
+ fieldsBuffer.writeVInt(info.number);
int bits = 0;
final BytesRef bytes;
final String string;
@@ -177,22 +190,22 @@ public final class Lucene40StoredFieldsW
}
}
- fieldsStream.writeByte((byte) bits);
+ fieldsBuffer.writeByte((byte) bits);
if (bytes != null) {
- fieldsStream.writeVInt(bytes.length);
- fieldsStream.writeBytes(bytes.bytes, bytes.offset, bytes.length);
+ fieldsBuffer.writeVInt(bytes.length);
+ fieldsBuffer.writeBytes(bytes.bytes, bytes.offset, bytes.length);
} else if (string != null) {
- fieldsStream.writeString(field.stringValue());
+ fieldsBuffer.writeString(field.stringValue());
} else {
if (number instanceof Byte || number instanceof Short || number instanceof Integer) {
- fieldsStream.writeInt(number.intValue());
+ fieldsBuffer.writeInt(number.intValue());
} else if (number instanceof Long) {
- fieldsStream.writeLong(number.longValue());
+ fieldsBuffer.writeLong(number.longValue());
} else if (number instanceof Float) {
- fieldsStream.writeInt(Float.floatToIntBits(number.floatValue()));
+ fieldsBuffer.writeInt(Float.floatToIntBits(number.floatValue()));
} else if (number instanceof Double) {
- fieldsStream.writeLong(Double.doubleToLongBits(number.doubleValue()));
+ fieldsBuffer.writeLong(Double.doubleToLongBits(number.doubleValue()));
} else {
throw new AssertionError("Cannot get here");
}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/Field.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/Field.java Mon Apr 28 17:31:13 2014
@@ -244,9 +244,23 @@ public class Field implements IndexableF
throw new IllegalArgumentException("it doesn't make sense to have a field that "
+ "is neither indexed nor stored");
}
- if (!type.indexed() && (type.storeTermVectors())) {
- throw new IllegalArgumentException("cannot store term vector information "
- + "for a field that is not indexed");
+ if (!type.indexed()) {
+ if (type.storeTermVectors()) {
+ throw new IllegalArgumentException("cannot store term vectors "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ if (type.storeTermVectorPositions()) {
+ throw new IllegalArgumentException("cannot store term vector positions "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ if (type.storeTermVectorOffsets()) {
+ throw new IllegalArgumentException("cannot store term vector offsets "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ if (type.storeTermVectorPayloads()) {
+ throw new IllegalArgumentException("cannot store term vector payloads "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
}
this.type = type;
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/FieldType.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/FieldType.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/document/FieldType.java Mon Apr 28 17:31:13 2014
@@ -382,9 +382,9 @@ public class FieldType implements Indexa
}
if (storeTermVectorPositions()) {
result.append(",termVectorPosition");
- if (storeTermVectorPayloads()) {
- result.append(",termVectorPayloads");
- }
+ }
+ if (storeTermVectorPayloads()) {
+ result.append(",termVectorPayloads");
}
if (omitNorms()) {
result.append(",omitNorms");
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/BinaryDocValuesWriter.java Mon Apr 28 17:31:13 2014
@@ -121,10 +121,6 @@ class BinaryDocValuesWriter extends DocV
});
}
- @Override
- public void abort() {
- }
-
// iterates over the values we have in ram
private class BytesIterator implements Iterator<BytesRef> {
final BytesRef value = new BytesRef();
Added: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java?rev=1590721&view=auto
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java (added)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java Mon Apr 28 17:31:13 2014
@@ -0,0 +1,669 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
+import org.apache.lucene.codecs.DocValuesConsumer;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosWriter;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.StoredFieldsWriter;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash.MaxBytesLengthExceededException;
+import org.apache.lucene.util.Counter;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
+
+/** Default general purpose indexing chain, which handles
+ * indexing all types of fields. */
+final class DefaultIndexingChain extends DocConsumer {
+ final Counter bytesUsed;
+ final DocumentsWriterPerThread.DocState docState;
+ final DocumentsWriterPerThread docWriter;
+ final FieldInfos.Builder fieldInfos;
+
+ // Writes postings and term vectors:
+ final TermsHash termsHash;
+
+ final StoredFieldsWriter storedFieldsWriter;
+ private int lastStoredDocID;
+
+ // NOTE: I tried using Hash Map<String,PerField>
+ // but it was ~2% slower on Wiki and Geonames with Java
+ // 1.7.0_25:
+ private PerField[] fieldHash = new PerField[2];
+ private int hashMask = 1;
+
+ private int totalFieldCount;
+ private long nextFieldGen;
+
+ // Lazy init:
+ private NumericDocValuesWriter norms;
+
+ // Holds fields seen in each document
+ private PerField[] fields = new PerField[1];
+
+ public DefaultIndexingChain(DocumentsWriterPerThread docWriter) throws IOException {
+ this.docWriter = docWriter;
+ this.fieldInfos = docWriter.getFieldInfosBuilder();
+ this.docState = docWriter.docState;
+ this.bytesUsed = docWriter.bytesUsed;
+
+ // Writes stored fields:
+ storedFieldsWriter = docWriter.codec.storedFieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegmentInfo(), IOContext.DEFAULT);
+
+ TermsHash termVectorsWriter = new TermVectorsConsumer(docWriter);
+ termsHash = new FreqProxTermsWriter(docWriter, termVectorsWriter);
+ }
+
+ @Override
+ public void flush(SegmentWriteState state) throws IOException {
+
+ // NOTE: caller (DocumentsWriterPerThread) handles
+ // aborting on any exception from this method
+
+ int numDocs = state.segmentInfo.getDocCount();
+ writeNorms(state);
+ writeDocValues(state);
+
+ fillStoredFields(numDocs);
+ storedFieldsWriter.finish(state.fieldInfos, numDocs);
+ storedFieldsWriter.close();
+
+ Map<String,TermsHashPerField> fieldsToFlush = new HashMap<>();
+ for (int i=0;i<fieldHash.length;i++) {
+ PerField perField = fieldHash[i];
+ while (perField != null) {
+ if (perField.invertState != null) {
+ fieldsToFlush.put(perField.fieldInfo.name, perField.termsHashPerField);
+ }
+ perField = perField.next;
+ }
+ }
+
+ termsHash.flush(fieldsToFlush, state);
+
+ // Important to save after asking consumer to flush so
+ // consumer can alter the FieldInfo* if necessary. EG,
+ // FreqProxTermsWriter does this with
+ // FieldInfo.storePayload.
+ FieldInfosWriter infosWriter = docWriter.codec.fieldInfosFormat().getFieldInfosWriter();
+ infosWriter.write(state.directory, state.segmentInfo.name, "", state.fieldInfos, IOContext.DEFAULT);
+ }
+
+ /** Writes all buffered doc values (called from {@link #flush}). */
+ private void writeDocValues(SegmentWriteState state) throws IOException {
+ int docCount = state.segmentInfo.getDocCount();
+ DocValuesConsumer dvConsumer = null;
+ boolean success = false;
+ try {
+ for (int i=0;i<fieldHash.length;i++) {
+ PerField perField = fieldHash[i];
+ while (perField != null) {
+ if (perField.docValuesWriter != null) {
+ if (dvConsumer == null) {
+ // lazy init
+ DocValuesFormat fmt = state.segmentInfo.getCodec().docValuesFormat();
+ dvConsumer = fmt.fieldsConsumer(state);
+ }
+
+ perField.docValuesWriter.finish(docCount);
+ perField.docValuesWriter.flush(state, dvConsumer);
+ perField.docValuesWriter = null;
+ }
+ perField = perField.next;
+ }
+ }
+
+ // TODO: catch missing DV fields here? else we have
+ // null/"" depending on how docs landed in segments?
+ // but we can't detect all cases, and we should leave
+ // this behavior undefined. dv is not "schemaless": its column-stride.
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(dvConsumer);
+ } else {
+ IOUtils.closeWhileHandlingException(dvConsumer);
+ }
+ }
+ }
+
+ /** Catch up for all docs before us that had no stored
+ * fields, or hit non-aborting exceptions before writing
+ * stored fields. */
+ private void fillStoredFields(int docID) throws IOException {
+ while (lastStoredDocID < docID) {
+ storedFieldsWriter.startDocument();
+ lastStoredDocID++;
+ storedFieldsWriter.finishDocument();
+ }
+ }
+
+ private void writeNorms(SegmentWriteState state) throws IOException {
+ boolean success = false;
+ DocValuesConsumer normsConsumer = null;
+ try {
+ if (state.fieldInfos.hasNorms()) {
+ NormsFormat normsFormat = state.segmentInfo.getCodec().normsFormat();
+ assert normsFormat != null;
+ normsConsumer = normsFormat.normsConsumer(state);
+
+ for (FieldInfo fi : state.fieldInfos) {
+ PerField perField = getPerField(fi.name);
+ assert perField != null;
+
+ // we must check the final value of omitNorms for the fieldinfo: it could have
+ // changed for this field since the first time we added it.
+ if (fi.omitsNorms() == false) {
+ if (perField.norms != null) {
+ perField.norms.finish(state.segmentInfo.getDocCount());
+ perField.norms.flush(state, normsConsumer);
+ assert fi.getNormType() == DocValuesType.NUMERIC;
+ } else if (fi.isIndexed()) {
+ assert fi.getNormType() == null: "got " + fi.getNormType() + "; field=" + fi.name;
+ }
+ }
+ }
+ }
+ success = true;
+ } finally {
+ if (success) {
+ IOUtils.close(normsConsumer);
+ } else {
+ IOUtils.closeWhileHandlingException(normsConsumer);
+ }
+ }
+ }
+
+ @Override
+ public void abort() {
+ try {
+ // E.g. close any open files in the stored fields writer:
+ storedFieldsWriter.abort();
+ } catch (Throwable t) {
+ }
+
+ try {
+ // E.g. close any open files in the term vectors writer:
+ termsHash.abort();
+ } catch (Throwable t) {
+ }
+
+ Arrays.fill(fieldHash, null);
+ }
+
+ private void rehash() {
+ int newHashSize = (fieldHash.length*2);
+ assert newHashSize > fieldHash.length;
+
+ PerField newHashArray[] = new PerField[newHashSize];
+
+ // Rehash
+ int newHashMask = newHashSize-1;
+ for(int j=0;j<fieldHash.length;j++) {
+ PerField fp0 = fieldHash[j];
+ while(fp0 != null) {
+ final int hashPos2 = fp0.fieldInfo.name.hashCode() & newHashMask;
+ PerField nextFP0 = fp0.next;
+ fp0.next = newHashArray[hashPos2];
+ newHashArray[hashPos2] = fp0;
+ fp0 = nextFP0;
+ }
+ }
+
+ fieldHash = newHashArray;
+ hashMask = newHashMask;
+ }
+
+ @Override
+ public void processDocument() throws IOException {
+
+ // How many indexed field names we've seen (collapses
+ // multiple field instances by the same name):
+ int fieldCount = 0;
+
+ long fieldGen = nextFieldGen++;
+
+ // NOTE: we need two passes here, in case there are
+ // multi-valued fields, because we must process all
+ // instances of a given field at once, since the
+ // analyzer is free to reuse TokenStream across fields
+ // (i.e., we cannot have more than one TokenStream
+ // running "at once"):
+
+ termsHash.startDocument();
+
+ // Invert indexed fields:
+ for (IndexableField field : docState.doc.indexableFields()) {
+ IndexableFieldType fieldType = field.fieldType();
+ PerField fp = getOrAddField(field.name(), fieldType, true);
+ boolean first = fp.fieldGen != fieldGen;
+ fp.invert(field, first);
+
+ if (first) {
+ fields[fieldCount++] = fp;
+ fp.fieldGen = fieldGen;
+ }
+ }
+
+ // Finish each field name seen in the document:
+ for (int i=0;i<fieldCount;i++) {
+ fields[i].finish();
+ }
+
+ boolean success = false;
+ try {
+ termsHash.finishDocument();
+ success = true;
+ } finally {
+ if (success == false) {
+ // Must abort, on the possibility that on-disk term
+ // vectors are now corrupt:
+ docWriter.setAborting();
+ }
+ }
+
+ // Add stored fields:
+ fillStoredFields(docState.docID);
+ storedFieldsWriter.startDocument();
+ lastStoredDocID++;
+
+ success = false;
+ try {
+ for (StorableField field : docState.doc.storableFields()) {
+ final String fieldName = field.name();
+ IndexableFieldType fieldType = field.fieldType();
+
+ verifyFieldType(fieldName, fieldType);
+
+ PerField fp = getOrAddField(fieldName, fieldType, false);
+ if (fieldType.stored()) {
+ storedFieldsWriter.writeField(fp.fieldInfo, field);
+ }
+
+ DocValuesType dvType = fieldType.docValueType();
+ if (dvType != null) {
+ indexDocValue(fp, dvType, field);
+ }
+ }
+ success = true;
+ } finally {
+ if (success == false) {
+ // We must abort, on the possibility that the
+ // stored fields file is now corrupt:
+ docWriter.setAborting();
+ } else {
+ storedFieldsWriter.finishDocument();
+ }
+ }
+ }
+
+ private static void verifyFieldType(String name, IndexableFieldType ft) {
+ if (ft.indexed() == false) {
+ if (ft.storeTermVectors()) {
+ throw new IllegalArgumentException("cannot store term vectors "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ if (ft.storeTermVectorPositions()) {
+ throw new IllegalArgumentException("cannot store term vector positions "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ if (ft.storeTermVectorOffsets()) {
+ throw new IllegalArgumentException("cannot store term vector offsets "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ if (ft.storeTermVectorPayloads()) {
+ throw new IllegalArgumentException("cannot store term vector payloads "
+ + "for a field that is not indexed (field=\"" + name + "\")");
+ }
+ }
+ }
+
+ /** Called from processDocument to index one field's doc
+ * value */
+ private void indexDocValue(PerField fp, DocValuesType dvType, StorableField field) throws IOException {
+
+ boolean hasDocValues = fp.fieldInfo.hasDocValues();
+
+ // This will throw an exc if the caller tried to
+ // change the DV type for the field:
+ fp.fieldInfo.setDocValuesType(dvType);
+ if (hasDocValues == false) {
+ fieldInfos.globalFieldNumbers.setDocValuesType(fp.fieldInfo.number, fp.fieldInfo.name, dvType);
+ }
+
+ int docID = docState.docID;
+
+ switch(dvType) {
+
+ case NUMERIC:
+ if (fp.docValuesWriter == null) {
+ fp.docValuesWriter = new NumericDocValuesWriter(fp.fieldInfo, bytesUsed, true);
+ }
+ ((NumericDocValuesWriter) fp.docValuesWriter).addValue(docID, field.numericValue().longValue());
+ break;
+
+ case BINARY:
+ if (fp.docValuesWriter == null) {
+ fp.docValuesWriter = new BinaryDocValuesWriter(fp.fieldInfo, bytesUsed);
+ }
+ ((BinaryDocValuesWriter) fp.docValuesWriter).addValue(docID, field.binaryValue());
+ break;
+
+ case SORTED:
+ if (fp.docValuesWriter == null) {
+ fp.docValuesWriter = new SortedDocValuesWriter(fp.fieldInfo, bytesUsed);
+ }
+ ((SortedDocValuesWriter) fp.docValuesWriter).addValue(docID, field.binaryValue());
+ break;
+
+ case SORTED_SET:
+ if (fp.docValuesWriter == null) {
+ fp.docValuesWriter = new SortedSetDocValuesWriter(fp.fieldInfo, bytesUsed);
+ }
+ ((SortedSetDocValuesWriter) fp.docValuesWriter).addValue(docID, field.binaryValue());
+ break;
+
+ default:
+ throw new AssertionError("unrecognized DocValues.Type: " + dvType);
+ }
+ }
+
+ /** Returns a previously created {@link PerField}, or null
+ * if this field name wasn't seen yet. */
+ private PerField getPerField(String name) {
+ final int hashPos = name.hashCode() & hashMask;
+ PerField fp = fieldHash[hashPos];
+ while (fp != null && !fp.fieldInfo.name.equals(name)) {
+ fp = fp.next;
+ }
+ return fp;
+ }
+
+ /** Returns a previously created {@link PerField},
+ * absorbing the type information from {@link FieldType},
+ * and creates a new {@link PerField} if this field name
+ * wasn't seen yet. */
+ private PerField getOrAddField(String name, IndexableFieldType fieldType, boolean invert) {
+
+ // Make sure we have a PerField allocated
+ final int hashPos = name.hashCode() & hashMask;
+ PerField fp = fieldHash[hashPos];
+ while (fp != null && !fp.fieldInfo.name.equals(name)) {
+ fp = fp.next;
+ }
+
+ if (fp == null) {
+ // First time we are seeing this field in this segment
+
+ FieldInfo fi = fieldInfos.addOrUpdate(name, fieldType);
+
+ fp = new PerField(fi, invert);
+ fp.next = fieldHash[hashPos];
+ fieldHash[hashPos] = fp;
+ totalFieldCount++;
+
+ // At most 50% load factor:
+ if (totalFieldCount >= fieldHash.length/2) {
+ rehash();
+ }
+
+ if (totalFieldCount > fields.length) {
+ PerField[] newFields = new PerField[ArrayUtil.oversize(totalFieldCount, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(fields, 0, newFields, 0, fields.length);
+ fields = newFields;
+ }
+
+ } else {
+ fp.fieldInfo.update(fieldType);
+
+ if (invert && fp.invertState == null) {
+ fp.setInvertState();
+ }
+ }
+
+ return fp;
+ }
+
+ /** NOTE: not static: accesses at least docState, termsHash. */
+ private final class PerField implements Comparable<PerField> {
+
+ final FieldInfo fieldInfo;
+ final Similarity similarity;
+
+ FieldInvertState invertState;
+ TermsHashPerField termsHashPerField;
+
+ // Non-null if this field ever had doc values in this
+ // segment:
+ DocValuesWriter docValuesWriter;
+
+ /** We use this to know when a PerField is seen for the
+ * first time in the current document. */
+ long fieldGen = -1;
+
+ // Used by the hash table
+ PerField next;
+
+ // Lazy init'd:
+ NumericDocValuesWriter norms;
+
+ public PerField(FieldInfo fieldInfo, boolean invert) {
+ this.fieldInfo = fieldInfo;
+ similarity = docState.similarity;
+ if (invert) {
+ setInvertState();
+ }
+ }
+
+ void setInvertState() {
+ invertState = new FieldInvertState(fieldInfo.name);
+ termsHashPerField = termsHash.addField(invertState, fieldInfo);
+ }
+
+ @Override
+ public int compareTo(PerField other) {
+ return this.fieldInfo.name.compareTo(other.fieldInfo.name);
+ }
+
+ public void finish() throws IOException {
+ if (fieldInfo.omitsNorms() == false) {
+ if (norms == null) {
+ fieldInfo.setNormValueType(FieldInfo.DocValuesType.NUMERIC);
+ norms = new NumericDocValuesWriter(fieldInfo, docState.docWriter.bytesUsed, false);
+ }
+ norms.addValue(docState.docID, similarity.computeNorm(invertState));
+ }
+
+ termsHashPerField.finish();
+ }
+
+ /** Inverts one field for one document; first is true
+ * if this is the first time we are seeing this field
+ * name in this document. */
+ public void invert(IndexableField field, boolean first) throws IOException {
+ if (first) {
+ // First time we're seeing this field (indexed) in
+ // this document:
+ invertState.reset();
+ }
+
+ IndexableFieldType fieldType = field.fieldType();
+
+ // if the field omits norms, the boost cannot be indexed.
+ if (fieldType.omitNorms() && field.boost() != 1.0f) {
+ throw new UnsupportedOperationException("You cannot set an index-time boost: norms are omitted for field '" + field.name() + "'");
+ }
+
+ final boolean analyzed = fieldType.tokenized() && docState.analyzer != null;
+
+ // only bother checking offsets if something will consume them.
+ // TODO: after we fix analyzers, also check if termVectorOffsets will be indexed.
+ final boolean checkOffsets = fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
+
+ int lastStartOffset = 0;
+
+ /*
+ * To assist people in tracking down problems in analysis components, we wish to write the field name to the infostream
+ * when we fail. We expect some caller to eventually deal with the real exception, so we don't want any 'catch' clauses,
+ * but rather a finally that takes note of the problem.
+ */
+ boolean aborting = false;
+ boolean succeededInProcessingField = false;
+ try (TokenStream stream = field.tokenStream(docState.analyzer)) {
+ // reset the TokenStream to the first token
+ stream.reset();
+
+ if (invertState.attributeSource != stream) {
+ // EmptyTokenStream gets angry otherwise:
+ if (stream.hasAttribute(TermToBytesRefAttribute.class)) {
+ invertState.termAttribute = stream.getAttribute(TermToBytesRefAttribute.class);
+ } else {
+ invertState.termAttribute = null;
+ }
+ invertState.posIncrAttribute = stream.addAttribute(PositionIncrementAttribute.class);
+ invertState.offsetAttribute = stream.addAttribute(OffsetAttribute.class);
+ if (stream.hasAttribute(PayloadAttribute.class)) {
+ invertState.payloadAttribute = stream.getAttribute(PayloadAttribute.class);
+ } else {
+ invertState.payloadAttribute = null;
+ }
+ invertState.attributeSource = stream;
+ }
+
+ termsHashPerField.start(field, first);
+
+ while (stream.incrementToken()) {
+
+ // If we hit an exception in stream.next below
+ // (which is fairly common, e.g. if analyzer
+ // chokes on a given document), then it's
+ // non-aborting and (above) this one document
+ // will be marked as deleted, but still
+ // consume a docID
+
+ final int posIncr = invertState.posIncrAttribute.getPositionIncrement();
+ if (posIncr < 0) {
+ throw new IllegalArgumentException("position increment must be >=0 (got " + posIncr + ") for field '" + field.name() + "'");
+ }
+ if (invertState.position == 0 && posIncr == 0) {
+ throw new IllegalArgumentException("first position increment must be > 0 (got 0) for field '" + field.name() + "'");
+ }
+ int position = invertState.position + posIncr;
+ if (position > 0) {
+ // NOTE: confusing: this "mirrors" the
+ // position++ we do below
+ position--;
+ } else if (position < 0) {
+ throw new IllegalArgumentException("position overflow for field '" + field.name() + "'");
+ }
+
+ // position is legal, we can safely place it in invertState now.
+ // not sure if anything will use invertState after non-aborting exc...
+ invertState.position = position;
+
+ if (posIncr == 0) {
+ invertState.numOverlap++;
+ }
+
+ if (checkOffsets) {
+ int startOffset = invertState.offset + invertState.offsetAttribute.startOffset();
+ int endOffset = invertState.offset + invertState.offsetAttribute.endOffset();
+ if (startOffset < 0 || endOffset < startOffset) {
+ throw new IllegalArgumentException("startOffset must be non-negative, and endOffset must be >= startOffset, "
+ + "startOffset=" + startOffset + ",endOffset=" + endOffset + " for field '" + field.name() + "'");
+ }
+ if (startOffset < lastStartOffset) {
+ throw new IllegalArgumentException("offsets must not go backwards startOffset="
+ + startOffset + " is < lastStartOffset=" + lastStartOffset + " for field '" + field.name() + "'");
+ }
+ lastStartOffset = startOffset;
+ }
+
+ //System.out.println(" term=" + invertState.termAttribute);
+
+ // If we hit an exception in here, we abort
+ // all buffered documents since the last
+ // flush, on the likelihood that the
+ // internal state of the terms hash is now
+ // corrupt and should not be flushed to a
+ // new segment:
+ aborting = true;
+ termsHashPerField.add();
+ aborting = false;
+
+ invertState.length++;
+ invertState.position++;
+ }
+
+ // trigger streams to perform end-of-stream operations
+ stream.end();
+
+ // TODO: maybe add some safety? then again, its already checked
+ // when we come back around to the field...
+ invertState.position += invertState.posIncrAttribute.getPositionIncrement();
+ invertState.offset += invertState.offsetAttribute.endOffset();
+
+ /* if there is an exception coming through, we won't set this to true here:*/
+ succeededInProcessingField = true;
+ } catch (MaxBytesLengthExceededException e) {
+ aborting = false;
+ byte[] prefix = new byte[30];
+ BytesRef bigTerm = invertState.termAttribute.getBytesRef();
+ System.arraycopy(bigTerm.bytes, bigTerm.offset, prefix, 0, 30);
+ String msg = "Document contains at least one immense term in field=\"" + fieldInfo.name + "\" (whose UTF8 encoding is longer than the max length " + DocumentsWriterPerThread.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + Arrays.toString(prefix) + "...'";
+ if (docState.infoStream.isEnabled("IW")) {
+ docState.infoStream.message("IW", "ERROR: " + msg);
+ }
+ // Document will be deleted above:
+ throw new IllegalArgumentException(msg);
+ } finally {
+ if (succeededInProcessingField == false && aborting) {
+ docState.docWriter.setAborting();
+ }
+
+ if (!succeededInProcessingField && docState.infoStream.isEnabled("DW")) {
+ docState.infoStream.message("DW", "An exception was thrown while processing field " + fieldInfo.name);
+ }
+ }
+
+ if (analyzed) {
+ invertState.position += docState.analyzer.getPositionIncrementGap(fieldInfo.name);
+ invertState.offset += docState.analyzer.getOffsetGap(fieldInfo.name);
+ }
+
+ invertState.boost *= field.boost();
+ }
+ }
+}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocConsumer.java Mon Apr 28 17:31:13 2014
@@ -20,8 +20,7 @@ package org.apache.lucene.index;
import java.io.IOException;
abstract class DocConsumer {
- abstract void processDocument(FieldInfos.Builder fieldInfos) throws IOException;
- abstract void finishDocument() throws IOException;
+ abstract void processDocument() throws IOException;
abstract void flush(final SegmentWriteState state) throws IOException;
abstract void abort();
}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocValuesWriter.java Mon Apr 28 17:31:13 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.codecs.DocValuesConsumer;
abstract class DocValuesWriter {
- abstract void abort() throws IOException;
abstract void finish(int numDoc);
abstract void flush(SegmentWriteState state, DocValuesConsumer consumer) throws IOException;
}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriter.java Mon Apr 28 17:31:13 2014
@@ -43,19 +43,13 @@ import org.apache.lucene.util.InfoStream
* This class accepts multiple added documents and directly
* writes segment files.
*
- * Each added document is passed to the {@link DocConsumer},
- * which in turn processes the document and interacts with
- * other consumers in the indexing chain. Certain
- * consumers, like {@link StoredFieldsConsumer} and {@link
- * TermVectorsConsumer}, digest a document and
- * immediately write bytes to the "doc store" files (ie,
- * they do not consume RAM per document, except while they
- * are processing the document).
+ * Each added document is passed to the indexing chain,
+ * which in turn processes the document into the different
+ * codec formats. Some formats write bytes to files
+ * immediately, e.g. stored fields and term vectors, while
+ * others are buffered by the indexing chain and written
+ * only on flush.
*
- * Other consumers, eg {@link FreqProxTermsWriter} and
- * {@link NormsConsumer}, buffer bytes in RAM and flush only
- * when a new segment is produced.
-
* Once we have used our allowed RAM buffer, or the number
* of added docs is large enough (in the case we are
* flushing by doc count instead of RAM usage), we create a
@@ -398,7 +392,7 @@ final class DocumentsWriter implements C
return hasEvents;
}
- private final void ensureInitialized(ThreadState state) {
+ private final void ensureInitialized(ThreadState state) throws IOException {
if (state.isActive() && state.dwpt == null) {
final FieldInfos.Builder infos = new FieldInfos.Builder(
writer.globalFieldNumberMap);
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java Mon Apr 28 17:31:13 2014
@@ -51,48 +51,15 @@ class DocumentsWriterPerThread {
* documents.
*/
abstract static class IndexingChain {
- abstract DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread);
+ abstract DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) throws IOException;
}
static final IndexingChain defaultIndexingChain = new IndexingChain() {
@Override
- DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) {
- /*
- This is the current indexing chain:
-
- DocConsumer / DocConsumerPerThread
- --> code: DocFieldProcessor
- --> DocFieldConsumer / DocFieldConsumerPerField
- --> code: DocFieldConsumers / DocFieldConsumersPerField
- --> code: DocInverter / DocInverterPerField
- --> InvertedDocConsumer / InvertedDocConsumerPerField
- --> code: TermsHash / TermsHashPerField
- --> TermsHashConsumer / TermsHashConsumerPerField
- --> code: FreqProxTermsWriter / FreqProxTermsWriterPerField
- --> code: TermVectorsTermsWriter / TermVectorsTermsWriterPerField
- --> InvertedDocEndConsumer / InvertedDocConsumerPerField
- --> code: NormsConsumer / NormsConsumerPerField
- --> StoredFieldsConsumer
- --> TwoStoredFieldConsumers
- -> code: StoredFieldsProcessor
- -> code: DocValuesProcessor
- */
-
- // Build up indexing chain:
-
- final TermsHashConsumer termVectorsWriter = new TermVectorsConsumer(documentsWriterPerThread);
- final TermsHashConsumer freqProxWriter = new FreqProxTermsWriter();
-
- final InvertedDocConsumer termsHash = new TermsHash(documentsWriterPerThread, freqProxWriter, true,
- new TermsHash(documentsWriterPerThread, termVectorsWriter, false, null));
- final NormsConsumer normsWriter = new NormsConsumer();
- final DocInverter docInverter = new DocInverter(documentsWriterPerThread.docState, termsHash, normsWriter);
- final StoredFieldsConsumer storedFields = new TwoStoredFieldsConsumers(
- new StoredFieldsProcessor(documentsWriterPerThread),
- new DocValuesProcessor(documentsWriterPerThread.bytesUsed));
- return new DocFieldProcessor(documentsWriterPerThread, docInverter, storedFields);
+ DocConsumer getChain(DocumentsWriterPerThread documentsWriterPerThread) throws IOException {
+ return new DefaultIndexingChain(documentsWriterPerThread);
}
};
@@ -103,7 +70,6 @@ class DocumentsWriterPerThread {
Similarity similarity;
int docID;
IndexDocument doc;
- String maxTermPrefix;
DocState(DocumentsWriterPerThread docWriter, InfoStream infoStream) {
this.docWriter = docWriter;
@@ -180,7 +146,7 @@ class DocumentsWriterPerThread {
boolean aborting = false; // True if an abort is pending
boolean hasAborted = false; // True if the last exception throws by #updateDocument was aborting
- private FieldInfos.Builder fieldInfos;
+ private final FieldInfos.Builder fieldInfos;
private final InfoStream infoStream;
private int numDocsInRAM;
final DocumentsWriterDeleteQueue deleteQueue;
@@ -192,7 +158,7 @@ class DocumentsWriterPerThread {
public DocumentsWriterPerThread(String segmentName, Directory directory, LiveIndexWriterConfig indexWriterConfig, InfoStream infoStream, DocumentsWriterDeleteQueue deleteQueue,
- FieldInfos.Builder fieldInfos) {
+ FieldInfos.Builder fieldInfos) throws IOException {
this.directoryOrig = directory;
this.directory = new TrackingDirectoryWrapper(directory);
this.fieldInfos = fieldInfos;
@@ -218,13 +184,16 @@ class DocumentsWriterPerThread {
// this should be the last call in the ctor
// it really sucks that we need to pull this within the ctor and pass this ref to the chain!
consumer = indexWriterConfig.getIndexingChain().getChain(this);
-
}
void setAborting() {
aborting = true;
}
+ public FieldInfos.Builder getFieldInfosBuilder() {
+ return fieldInfos;
+ }
+
boolean checkAndResetHasAborted() {
final boolean retval = hasAborted;
hasAborted = false;
@@ -250,7 +219,7 @@ class DocumentsWriterPerThread {
boolean success = false;
try {
try {
- consumer.processDocument(fieldInfos);
+ consumer.processDocument();
} finally {
docState.clear();
}
@@ -266,15 +235,6 @@ class DocumentsWriterPerThread {
}
}
}
- success = false;
- try {
- consumer.finishDocument();
- success = true;
- } finally {
- if (!success) {
- abort(filesToDelete);
- }
- }
finishDocument(delTerm);
}
@@ -296,7 +256,7 @@ class DocumentsWriterPerThread {
boolean success = false;
try {
- consumer.processDocument(fieldInfos);
+ consumer.processDocument();
success = true;
} finally {
if (!success) {
@@ -310,16 +270,6 @@ class DocumentsWriterPerThread {
}
}
}
- success = false;
- try {
- consumer.finishDocument();
- success = true;
- } finally {
- if (!success) {
- abort(filesToDelete);
- }
- }
-
finishDocument(null);
}
allDocsIndexed = true;
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInfo.java Mon Apr 28 17:31:13 2014
@@ -169,16 +169,10 @@ public final class FieldInfo {
// should only be called by FieldInfos#addOrUpdate
void update(boolean indexed, boolean storeTermVector, boolean omitNorms, boolean storePayloads, IndexOptions indexOptions) {
//System.out.println("FI.update field=" + name + " indexed=" + indexed + " omitNorms=" + omitNorms + " this.omitNorms=" + this.omitNorms);
- if (this.indexed != indexed) {
- this.indexed = true; // once indexed, always index
- }
+ this.indexed |= indexed; // once indexed, always indexed
if (indexed) { // if updated field data is not for indexing, leave the updates out
- if (this.storeTermVector != storeTermVector) {
- this.storeTermVector = true; // once vector, always vector
- }
- if (this.storePayloads != storePayloads) {
- this.storePayloads = true;
- }
+ this.storeTermVector |= storeTermVector; // once vector, always vector
+ this.storePayloads |= storePayloads;
if (this.omitNorms != omitNorms) {
this.omitNorms = true; // if one require omitNorms at least once, it remains off for life
this.normType = null;
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FieldInvertState.java Mon Apr 28 17:31:13 2014
@@ -17,6 +17,10 @@
package org.apache.lucene.index;
import org.apache.lucene.analysis.TokenStream; // javadocs
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
import org.apache.lucene.util.AttributeSource;
/**
@@ -37,6 +41,11 @@ public final class FieldInvertState {
float boost;
AttributeSource attributeSource;
+ OffsetAttribute offsetAttribute;
+ PositionIncrementAttribute posIncrAttribute;
+ PayloadAttribute payloadAttribute;
+ TermToBytesRefAttribute termAttribute;
+
/** Creates {code FieldInvertState} for the specified
* field name. */
public FieldInvertState(String name) {
@@ -65,7 +74,6 @@ public final class FieldInvertState {
maxTermFrequency = 0;
uniqueTermCount = 0;
boost = 1.0f;
- attributeSource = null;
}
/**
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java Mon Apr 28 17:31:13 2014
@@ -126,7 +126,7 @@ class FreqProxFields extends Fields {
@Override
public boolean hasPayloads() {
- return terms.hasPayloads;
+ return terms.sawPayloads;
}
}
@@ -140,10 +140,10 @@ class FreqProxFields extends Fields {
public FreqProxTermsEnum(FreqProxTermsWriterPerField terms) {
this.terms = terms;
- this.numTerms = terms.termsHashPerField.bytesHash.size();
+ this.numTerms = terms.bytesHash.size();
sortedTermIDs = terms.sortedTermIDs;
assert sortedTermIDs != null;
- postingsArray = (FreqProxPostingsArray) terms.termsHashPerField.postingsArray;
+ postingsArray = (FreqProxPostingsArray) terms.postingsArray;
}
public void reset() {
@@ -161,7 +161,7 @@ class FreqProxFields extends Fields {
while (hi >= lo) {
int mid = (lo + hi) >>> 1;
int textStart = postingsArray.textStarts[sortedTermIDs[mid]];
- terms.termsHashPerField.bytePool.setBytesRef(scratch, textStart);
+ terms.bytePool.setBytesRef(scratch, textStart);
int cmp = scratch.compareTo(text);
if (cmp < 0) {
lo = mid + 1;
@@ -186,7 +186,7 @@ class FreqProxFields extends Fields {
public void seekExact(long ord) {
this.ord = (int) ord;
int textStart = postingsArray.textStarts[sortedTermIDs[this.ord]];
- terms.termsHashPerField.bytePool.setBytesRef(scratch, textStart);
+ terms.bytePool.setBytesRef(scratch, textStart);
}
@Override
@@ -196,7 +196,7 @@ class FreqProxFields extends Fields {
return null;
} else {
int textStart = postingsArray.textStarts[sortedTermIDs[ord]];
- terms.termsHashPerField.bytePool.setBytesRef(scratch, textStart);
+ terms.bytePool.setBytesRef(scratch, textStart);
return scratch;
}
}
@@ -324,7 +324,7 @@ class FreqProxFields extends Fields {
public void reset(int termID) {
this.termID = termID;
- terms.termsHashPerField.initReader(reader, termID, 0);
+ terms.initReader(reader, termID, 0);
ended = false;
docID = 0;
}
@@ -415,8 +415,8 @@ class FreqProxFields extends Fields {
public void reset(int termID) {
this.termID = termID;
- terms.termsHashPerField.initReader(reader, termID, 0);
- terms.termsHashPerField.initReader(posReader, termID, 1);
+ terms.initReader(reader, termID, 0);
+ terms.initReader(posReader, termID, 1);
ended = false;
docID = 0;
posLeft = 0;
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Mon Apr 28 17:31:13 2014
@@ -23,13 +23,13 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
-final class FreqProxTermsWriter extends TermsHashConsumer {
+final class FreqProxTermsWriter extends TermsHash {
- @Override
- void abort() {}
+ public FreqProxTermsWriter(DocumentsWriterPerThread docWriter, TermsHash termVectors) {
+ super(docWriter, true, termVectors);
+ }
private void applyDeletes(SegmentWriteState state, Fields fields) throws IOException {
// Process any pending Term deletes for this newly
@@ -77,21 +77,16 @@ final class FreqProxTermsWriter extends
}
}
- // TODO: would be nice to factor out more of this, eg the
- // FreqProxFieldMergeState, and code to visit all Fields
- // under the same FieldInfo together, up into TermsHash*.
- // Other writers would presumably share alot of this...
-
@Override
- public void flush(Map<String,TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ public void flush(Map<String,TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ super.flush(fieldsToFlush, state);
- // Gather all FieldData's that have postings, across all
- // ThreadStates
+ // Gather all fields that saw any postings:
List<FreqProxTermsWriterPerField> allFields = new ArrayList<>();
- for (TermsHashConsumerPerField f : fieldsToFlush.values()) {
+ for (TermsHashPerField f : fieldsToFlush.values()) {
final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f;
- if (perField.termsHashPerField.bytesHash.size() > 0) {
+ if (perField.bytesHash.size() > 0) {
perField.sortPostings();
assert perField.fieldInfo.isIndexed();
allFields.add(perField);
@@ -108,18 +103,8 @@ final class FreqProxTermsWriter extends
state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state).write(fields);
}
- BytesRef payload;
-
- @Override
- public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo) {
- return new FreqProxTermsWriterPerField(termsHashPerField, this, fieldInfo);
- }
-
- @Override
- void finishDocument(TermsHash termsHash) {
- }
-
@Override
- void startDocument() {
+ public TermsHashPerField addField(FieldInvertState invertState, FieldInfo fieldInfo) {
+ return new FreqProxTermsWriterPerField(invertState, this, fieldInfo, nextTermsHash.addField(invertState, fieldInfo));
}
}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon Apr 28 17:31:13 2014
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.IOException;
+
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.index.FieldInfo.IndexOptions;
@@ -26,16 +28,13 @@ import org.apache.lucene.util.RamUsageEs
// TODO: break into separate freq and prox writers as
// codecs; make separate container (tii/tis/skip/*) that can
// be configured as any number of files 1..N
-final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implements Comparable<FreqProxTermsWriterPerField> {
+final class FreqProxTermsWriterPerField extends TermsHashPerField {
+
+ private FreqProxPostingsArray freqProxPostingsArray;
- final FreqProxTermsWriter parent;
- final TermsHashPerField termsHashPerField;
- final FieldInfo fieldInfo;
- final DocumentsWriterPerThread.DocState docState;
- final FieldInvertState fieldState;
- boolean hasFreq;
- boolean hasProx;
- boolean hasOffsets;
+ final boolean hasFreq;
+ final boolean hasProx;
+ final boolean hasOffsets;
PayloadAttribute payloadAttribute;
OffsetAttribute offsetAttribute;
long sumTotalTermFreq;
@@ -44,111 +43,67 @@ final class FreqProxTermsWriterPerField
// How many docs have this field:
int docCount;
- public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriter parent, FieldInfo fieldInfo) {
- this.termsHashPerField = termsHashPerField;
- this.parent = parent;
- this.fieldInfo = fieldInfo;
- docState = termsHashPerField.docState;
- fieldState = termsHashPerField.fieldState;
- setIndexOptions(fieldInfo.getIndexOptions());
- }
-
- @Override
- int getStreamCount() {
- if (!hasProx) {
- return 1;
- } else {
- return 2;
- }
+ /** Set to true if any token had a payload in the current
+ * segment. */
+ boolean sawPayloads;
+
+ public FreqProxTermsWriterPerField(FieldInvertState invertState, TermsHash termsHash, FieldInfo fieldInfo, TermsHashPerField nextPerField) {
+ super(fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 ? 2 : 1, invertState, termsHash, nextPerField, fieldInfo);
+ IndexOptions indexOptions = fieldInfo.getIndexOptions();
+ assert indexOptions != null;
+ hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
}
@Override
- void finish() {
+ void finish() throws IOException {
+ super.finish();
sumDocFreq += fieldState.uniqueTermCount;
sumTotalTermFreq += fieldState.length;
if (fieldState.length > 0) {
docCount++;
}
- if (hasPayloads) {
+ if (sawPayloads) {
fieldInfo.setStorePayloads();
}
}
- boolean hasPayloads;
-
- @Override
- void skippingLongTerm() {}
-
@Override
- public int compareTo(FreqProxTermsWriterPerField other) {
- return fieldInfo.name.compareTo(other.fieldInfo.name);
- }
-
- private void setIndexOptions(IndexOptions indexOptions) {
- if (indexOptions == null) {
- // field could later be updated with indexed=true, so set everything on
- hasFreq = hasProx = hasOffsets = true;
- } else {
- hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- }
- }
-
- @Override
- boolean start(IndexableField[] fields, int count) {
+ boolean start(IndexableField f, boolean first) {
+ super.start(f, first);
+ payloadAttribute = fieldState.payloadAttribute;
+ offsetAttribute = fieldState.offsetAttribute;
return true;
}
- @Override
- void start(IndexableField f) {
- if (fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
- payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
- } else {
- payloadAttribute = null;
- }
- if (hasOffsets) {
- offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
- } else {
- offsetAttribute = null;
- }
- }
-
- void writeProx(final int termID, int proxCode) {
- //System.out.println("writeProx termID=" + termID + " proxCode=" + proxCode);
- assert hasProx;
- final BytesRef payload;
+ void writeProx(int termID, int proxCode) {
if (payloadAttribute == null) {
- payload = null;
- } else {
- payload = payloadAttribute.getPayload();
- }
-
- if (payload != null && payload.length > 0) {
- termsHashPerField.writeVInt(1, (proxCode<<1)|1);
- termsHashPerField.writeVInt(1, payload.length);
- termsHashPerField.writeBytes(1, payload.bytes, payload.offset, payload.length);
- hasPayloads = true;
+ writeVInt(1, proxCode<<1);
} else {
- termsHashPerField.writeVInt(1, proxCode<<1);
+ BytesRef payload = payloadAttribute.getPayload();
+ if (payload != null && payload.length > 0) {
+ writeVInt(1, (proxCode<<1)|1);
+ writeVInt(1, payload.length);
+ writeBytes(1, payload.bytes, payload.offset, payload.length);
+ sawPayloads = true;
+ } else {
+ writeVInt(1, proxCode<<1);
+ }
}
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
- postings.lastPositions[termID] = fieldState.position;
+ assert postingsArray == freqProxPostingsArray;
+ freqProxPostingsArray.lastPositions[termID] = fieldState.position;
}
- void writeOffsets(final int termID, int offsetAccum) {
- assert hasOffsets;
+ void writeOffsets(int termID, int offsetAccum) {
final int startOffset = offsetAccum + offsetAttribute.startOffset();
final int endOffset = offsetAccum + offsetAttribute.endOffset();
- //System.out.println("writeOffsets termID=" + termID + " prevOffset=" + prevOffset + " startOff=" + startOffset + " endOff=" + endOffset);
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
- assert startOffset - postings.lastOffsets[termID] >= 0;
- termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
-
- postings.lastOffsets[termID] = startOffset;
+ assert startOffset - freqProxPostingsArray.lastOffsets[termID] >= 0;
+ writeVInt(1, startOffset - freqProxPostingsArray.lastOffsets[termID]);
+ writeVInt(1, endOffset - startOffset);
+ freqProxPostingsArray.lastOffsets[termID] = startOffset;
}
@Override
@@ -157,9 +112,11 @@ final class FreqProxTermsWriterPerField
// flush
assert docState.testPoint("FreqProxTermsWriterPerField.newTerm start");
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
+ final FreqProxPostingsArray postings = freqProxPostingsArray;
+
postings.lastDocIDs[termID] = docState.docID;
if (!hasFreq) {
+ assert postings.termFreqs == null;
postings.lastDocCodes[termID] = docState.docID;
} else {
postings.lastDocCodes[termID] = docState.docID << 1;
@@ -182,15 +139,16 @@ final class FreqProxTermsWriterPerField
assert docState.testPoint("FreqProxTermsWriterPerField.addTerm start");
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
+ final FreqProxPostingsArray postings = freqProxPostingsArray;
assert !hasFreq || postings.termFreqs[termID] > 0;
if (!hasFreq) {
assert postings.termFreqs == null;
if (docState.docID != postings.lastDocIDs[termID]) {
+ // New document; now encode docCode for previous doc:
assert docState.docID > postings.lastDocIDs[termID];
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
+ writeVInt(0, postings.lastDocCodes[termID]);
postings.lastDocCodes[termID] = docState.docID - postings.lastDocIDs[termID];
postings.lastDocIDs[termID] = docState.docID;
fieldState.uniqueTermCount++;
@@ -203,11 +161,13 @@ final class FreqProxTermsWriterPerField
// Now that we know doc freq for previous doc,
// write it & lastDocCode
if (1 == postings.termFreqs[termID]) {
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]|1);
+ writeVInt(0, postings.lastDocCodes[termID]|1);
} else {
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
- termsHashPerField.writeVInt(0, postings.termFreqs[termID]);
+ writeVInt(0, postings.lastDocCodes[termID]);
+ writeVInt(0, postings.termFreqs[termID]);
}
+
+ // Init freq for the current document
postings.termFreqs[termID] = 1;
fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1;
@@ -226,15 +186,25 @@ final class FreqProxTermsWriterPerField
fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.termFreqs[termID]);
if (hasProx) {
writeProx(termID, fieldState.position-postings.lastPositions[termID]);
- }
- if (hasOffsets) {
- writeOffsets(termID, fieldState.offset);
+ if (hasOffsets) {
+ writeOffsets(termID, fieldState.offset);
+ }
}
}
}
@Override
+ public void newPostingsArray() {
+ freqProxPostingsArray = (FreqProxPostingsArray) postingsArray;
+ }
+
+ @Override
ParallelPostingsArray createPostingsArray(int size) {
+ IndexOptions indexOptions = fieldInfo.getIndexOptions();
+ assert indexOptions != null;
+ boolean hasFreq = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ boolean hasProx = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ boolean hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
return new FreqProxPostingsArray(size, hasFreq, hasProx, hasOffsets);
}
@@ -307,15 +277,4 @@ final class FreqProxTermsWriterPerField
return bytes;
}
}
-
- public void abort() {}
-
- BytesRef payload;
-
- int[] sortedTermIDs;
-
- void sortPostings() {
- assert sortedTermIDs == null;
- sortedTermIDs = termsHashPerField.sortPostings();
- }
}
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/IndexableFieldType.java Mon Apr 28 17:31:13 2014
@@ -39,6 +39,8 @@ public interface IndexableFieldType {
* <p>
* This has no effect if {@link #indexed()} returns false.
*/
+ // TODO: shouldn't we remove this? Whether/how a field is
+ // tokenized is an impl detail under Field?
public boolean tokenized();
/**
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/NumericDocValuesWriter.java Mon Apr 28 17:31:13 2014
@@ -97,10 +97,6 @@ class NumericDocValuesWriter extends Doc
});
}
- @Override
- public void abort() {
- }
-
// iterates over the values we have in ram
private class NumericIterator implements Iterator<Number> {
final AppendingDeltaPackedLongBuffer.Iterator iter = pending.iterator();
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedDocValuesWriter.java Mon Apr 28 17:31:13 2014
@@ -139,10 +139,6 @@ class SortedDocValuesWriter extends DocV
});
}
- @Override
- public void abort() {
- }
-
// iterates over the unique values we have in ram
private class ValuesIterator implements Iterator<BytesRef> {
final int sortedValues[];
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/SortedSetDocValuesWriter.java Mon Apr 28 17:31:13 2014
@@ -187,10 +187,6 @@ class SortedSetDocValuesWriter extends D
});
}
- @Override
- public void abort() {
- }
-
// iterates over the unique values we have in ram
private class ValuesIterator implements Iterator<BytesRef> {
final int sortedValues[];
Modified: lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java?rev=1590721&r1=1590720&r2=1590721&view=diff
==============================================================================
--- lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java (original)
+++ lucene/dev/branches/lucene5611/lucene/core/src/java/org/apache/lucene/index/TermVectorsConsumer.java Mon Apr 28 17:31:13 2014
@@ -29,28 +29,32 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
-final class TermVectorsConsumer extends TermsHashConsumer {
+final class TermVectorsConsumer extends TermsHash {
TermVectorsWriter writer;
- final DocumentsWriterPerThread docWriter;
- final DocumentsWriterPerThread.DocState docState;
+
+ /** Scratch term used by TermVectorsConsumerPerField.finishDocument. */
final BytesRef flushTerm = new BytesRef();
- // Used by perField when serializing the term vectors
+ final DocumentsWriterPerThread docWriter;
+
+ /** Used by TermVectorsConsumerPerField when serializing
+ * the term vectors. */
final ByteSliceReader vectorSliceReaderPos = new ByteSliceReader();
final ByteSliceReader vectorSliceReaderOff = new ByteSliceReader();
+
boolean hasVectors;
int numVectorFields;
int lastDocID;
private TermVectorsConsumerPerField[] perFields = new TermVectorsConsumerPerField[1];
public TermVectorsConsumer(DocumentsWriterPerThread docWriter) {
+ super(docWriter, false, null);
this.docWriter = docWriter;
- docState = docWriter.docState;
}
@Override
- void flush(Map<String, TermsHashConsumerPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ void flush(Map<String, TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
if (writer != null) {
int numDocs = state.segmentInfo.getDocCount();
assert numDocs > 0;
@@ -78,7 +82,7 @@ final class TermVectorsConsumer extends
}
}
- private final void initTermVectorsWriter() throws IOException {
+ private void initTermVectorsWriter() throws IOException {
if (writer == null) {
IOContext context = new IOContext(new FlushInfo(docWriter.getNumDocsInRAM(), docWriter.bytesUsed()));
writer = docWriter.codec.termVectorsFormat().vectorsWriter(docWriter.directory, docWriter.getSegmentInfo(), context);
@@ -87,7 +91,7 @@ final class TermVectorsConsumer extends
}
@Override
- void finishDocument(TermsHash termsHash) throws IOException {
+ void finishDocument() throws IOException {
assert docWriter.testPoint("TermVectorsTermsWriter.finishDocument start");
@@ -95,6 +99,9 @@ final class TermVectorsConsumer extends
return;
}
+ // Fields in term vectors are UTF16 sorted:
+ ArrayUtil.introSort(perFields, 0, numVectorFields);
+
initTermVectorsWriter();
fill(docState.docID);
@@ -110,32 +117,35 @@ final class TermVectorsConsumer extends
lastDocID++;
- termsHash.reset();
- reset();
+ super.reset();
+ resetFields();
assert docWriter.testPoint("TermVectorsTermsWriter.finishDocument end");
}
@Override
public void abort() {
hasVectors = false;
+ try {
+ super.abort();
+ } finally {
+ if (writer != null) {
+ writer.abort();
+ writer = null;
+ }
- if (writer != null) {
- writer.abort();
- writer = null;
+ lastDocID = 0;
+ reset();
}
-
- lastDocID = 0;
- reset();
}
- void reset() {
- Arrays.fill(perFields, null);// don't hang onto stuff from previous doc
+ void resetFields() {
+ Arrays.fill(perFields, null); // don't hang onto stuff from previous doc
numVectorFields = 0;
}
@Override
- public TermsHashConsumerPerField addField(TermsHashPerField termsHashPerField, FieldInfo fieldInfo) {
- return new TermVectorsConsumerPerField(termsHashPerField, this, fieldInfo);
+ public TermsHashPerField addField(FieldInvertState invertState, FieldInfo fieldInfo) {
+ return new TermVectorsConsumerPerField(invertState, this, fieldInfo);
}
void addFieldToFlush(TermVectorsConsumerPerField fieldToFlush) {
@@ -151,24 +161,7 @@ final class TermVectorsConsumer extends
@Override
void startDocument() {
- assert clearLastVectorFieldName();
- reset();
- }
-
- // Called only by assert
- final boolean clearLastVectorFieldName() {
- lastVectorFieldName = null;
- return true;
- }
-
- // Called only by assert
- String lastVectorFieldName;
- final boolean vectorFieldsInOrder(FieldInfo fi) {
- try {
- return lastVectorFieldName != null ? lastVectorFieldName.compareTo(fi.name) < 0 : true;
- } finally {
- lastVectorFieldName = fi.name;
- }
+ resetFields();
+ numVectorFields = 0;
}
-
}