You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/08/25 03:52:09 UTC
svn commit: r1620250 [2/4] - in /lucene/dev/branches/branch_4x/lucene: ./
codecs/src/java/org/apache/lucene/codecs/blockterms/
codecs/src/java/org/apache/lucene/codecs/blocktreeords/
codecs/src/java/org/apache/lucene/codecs/bloom/ codecs/src/java/org/a...
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/FieldsConsumer.java Mon Aug 25 01:52:08 2014
@@ -20,34 +20,26 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import org.apache.lucene.index.AtomicReader;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.MappedMultiFields;
import org.apache.lucene.index.MergeState;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.ReaderSlice;
-import org.apache.lucene.index.SegmentWriteState; // javadocs
-import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
/**
* Abstract API that consumes terms, doc, freq, prox, offset and
* payloads postings. Concrete implementations of this
* actually do "something" with the postings (write it into
* the index in a specific format).
- * <p>
- * The lifecycle is:
- * <ol>
- * <li>FieldsConsumer is created by
- * {@link PostingsFormat#fieldsConsumer(SegmentWriteState)}.
- * <li>For each field, {@link #addField(FieldInfo)} is called,
- * returning a {@link TermsConsumer} for the field.
- * <li>After all fields are added, the consumer is {@link #close}d.
- * </ol>
*
* @lucene.experimental
*/
+
public abstract class FieldsConsumer implements Closeable {
/** Sole constructor. (For invocation by subclass
@@ -55,18 +47,45 @@ public abstract class FieldsConsumer imp
protected FieldsConsumer() {
}
- /** Add a new field */
- public abstract TermsConsumer addField(FieldInfo field) throws IOException;
-
- /** Called when we are done adding everything. */
- @Override
- public abstract void close() throws IOException;
+ // TODO: can we somehow compute stats for you...?
- /** Called during merging to merge all {@link Fields} from
- * sub-readers. This must recurse to merge all postings
- * (terms, docs, positions, etc.). A {@link
- * PostingsFormat} can override this default
- * implementation to do its own merging. */
+ // TODO: maybe we should factor out "limited" (only
+ // iterables, no counts/stats) base classes from
+ // Fields/Terms/Docs/AndPositions?
+
+ /** Write all fields, terms and postings. This the "pull"
+ * API, allowing you to iterate more than once over the
+ * postings, somewhat analogous to using a DOM API to
+ * traverse an XML tree.
+ *
+ * <p><b>Notes</b>:
+ *
+ * <ul>
+ * <li> You must compute index statistics,
+ * including each Term's docFreq and totalTermFreq,
+ * as well as the summary sumTotalTermFreq,
+ * sumTotalDocFreq and docCount.
+ *
+ * <li> You must skip terms that have no docs and
+ * fields that have no terms, even though the provided
+ * Fields API will expose them; this typically
+ * requires lazily writing the field or term until
+ * you've actually seen the first term or
+ * document.
+ *
+ * <li> The provided Fields instance is limited: you
+ * cannot call any methods that return
+ * statistics/counts; you cannot pass a non-null
+ * live docs when pulling docs/positions enums.
+ * </ul>
+ */
+ public abstract void write(Fields fields) throws IOException;
+
+ /** Merges in the fields from the readers in
+ * <code>mergeState</code>. The default implementation skips
+ * and maps around deleted documents, and calls {@link #write(Fields)}.
+ * Implementations can override this method for more sophisticated
+ * merging (bulk-byte copying, etc). */
public void merge(MergeState mergeState) throws IOException {
final List<Fields> fields = new ArrayList<>();
final List<ReaderSlice> slices = new ArrayList<>();
@@ -83,18 +102,17 @@ public abstract class FieldsConsumer imp
}
docBase += maxDoc;
}
-
- final Fields mergedFields = new MultiFields(fields.toArray(Fields.EMPTY_ARRAY),
- slices.toArray(ReaderSlice.EMPTY_ARRAY));
-
- for (String field : mergedFields) {
- FieldInfo info = mergeState.fieldInfos.fieldInfo(field);
- assert info != null : "FieldInfo for field is null: "+ field;
- Terms terms = mergedFields.terms(field);
- if (terms != null) {
- final TermsConsumer termsConsumer = addField(info);
- termsConsumer.merge(mergeState, info.getIndexOptions(), terms.iterator(null));
- }
- }
+
+ Fields mergedFields = new MappedMultiFields(mergeState,
+ new MultiFields(fields.toArray(Fields.EMPTY_ARRAY),
+ slices.toArray(ReaderSlice.EMPTY_ARRAY)));
+ write(mergedFields);
}
+
+ /** Return the BytesRef Comparator used to sort terms */
+ public abstract Comparator<BytesRef> getComparator();
+
+ // NOTE: strange but necessary so javadocs linting is happy:
+ @Override
+ public abstract void close() throws IOException;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java Mon Aug 25 01:52:08 2014
@@ -17,30 +17,30 @@ package org.apache.lucene.codecs;
* limitations under the License.
*/
-import java.io.IOException;
import java.io.Closeable;
+import java.io.IOException;
import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
+import org.apache.lucene.index.DocsAndPositionsEnum; // javadocs
+import org.apache.lucene.index.DocsEnum; // javadocs
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
/**
- * Extension of {@link PostingsConsumer} to support pluggable term dictionaries.
- * <p>
- * This class contains additional hooks to interact with the provided
- * term dictionaries such as {@link BlockTreeTermsWriter}. If you want
- * to re-use an existing implementation and are only interested in
- * customizing the format of the postings list, extend this class
- * instead.
+ * Class that plugs into term dictionaries, such as {@link
+ * BlockTreeTermsWriter}, and handles writing postings.
*
* @see PostingsReaderBase
* @lucene.experimental
*/
// TODO: find a better name; this defines the API that the
// terms dict impls use to talk to a postings impl.
-// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer
-public abstract class PostingsWriterBase extends PostingsConsumer implements Closeable {
+// TermsDict + PostingsReader/WriterBase == FieldsProducer/Consumer
+public abstract class PostingsWriterBase implements Closeable {
/** Sole constructor. (For invocation by subclass
* constructors, typically implicit.) */
@@ -52,18 +52,16 @@ public abstract class PostingsWriterBase
* the provided {@code termsOut}. */
public abstract void init(IndexOutput termsOut) throws IOException;
- /** Return a newly created empty TermState */
- public abstract BlockTermState newTermState() throws IOException;
-
- /** Start a new term. Note that a matching call to {@link
- * #finishTerm(BlockTermState)} is done, only if the term has at least one
- * document. */
- public abstract void startTerm() throws IOException;
-
- /** Finishes the current term. The provided {@link
- * BlockTermState} contains the term's summary statistics,
- * and will holds metadata from PBF when returned */
- public abstract void finishTerm(BlockTermState state) throws IOException;
+ /** Write all postings for one term; use the provided
+ * {@link TermsEnum} to pull a {@link DocsEnum} or {@link
+ * DocsAndPositionsEnum}. This method should not
+ * re-position the {@code TermsEnum}! It is already
+ * positioned on the term that should be written. This
+ * method must set the bit in the provided {@link
+ * FixedBitSet} for every docID written. If no docs
+ * were written, this method should return null, and the
+ * terms dict will skip the term. */
+ public abstract BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen) throws IOException;
/**
* Encode metadata as long[] and byte[]. {@code absolute} controls whether
Added: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java?rev=1620250&view=auto
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java (added)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/PushPostingsWriterBase.java Mon Aug 25 01:52:08 2014
@@ -0,0 +1,219 @@
+package org.apache.lucene.codecs;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
+
+/**
+ * Extension of {@link PostingsWriterBase}, adding a push
+ * API for writing each element of the postings. This API
+ * is somewhat analagous to an XML SAX API, while {@link
+ * PostingsWriterBase} is more like an XML DOM API.
+ *
+ * @see PostingsReaderBase
+ * @lucene.experimental
+ */
+// TODO: find a better name; this defines the API that the
+// terms dict impls use to talk to a postings impl.
+// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer
+public abstract class PushPostingsWriterBase extends PostingsWriterBase {
+
+ // Reused in writeTerm
+ private DocsEnum docsEnum;
+ private DocsAndPositionsEnum posEnum;
+ private int enumFlags;
+
+ /** {@link FieldInfo} of current field being written. */
+ protected FieldInfo fieldInfo;
+
+ /** {@link IndexOptions} of current field being
+ written */
+ protected IndexOptions indexOptions;
+
+ /** True if the current field writes freqs. */
+ protected boolean writeFreqs;
+
+ /** True if the current field writes positions. */
+ protected boolean writePositions;
+
+ /** True if the current field writes payloads. */
+ protected boolean writePayloads;
+
+ /** True if the current field writes offsets. */
+ protected boolean writeOffsets;
+
+ /** Sole constructor. (For invocation by subclass
+ * constructors, typically implicit.) */
+ protected PushPostingsWriterBase() {
+ }
+
+ /** Called once after startup, before any terms have been
+ * added. Implementations typically write a header to
+ * the provided {@code termsOut}. */
+ public abstract void init(IndexOutput termsOut) throws IOException;
+
+ /** Return a newly created empty TermState */
+ public abstract BlockTermState newTermState() throws IOException;
+
+ /** Start a new term. Note that a matching call to {@link
+ * #finishTerm(BlockTermState)} is done, only if the term has at least one
+ * document. */
+ public abstract void startTerm() throws IOException;
+
+ /** Finishes the current term. The provided {@link
+ * BlockTermState} contains the term's summary statistics,
+ * and will holds metadata from PBF when returned */
+ public abstract void finishTerm(BlockTermState state) throws IOException;
+
+ /**
+ * Encode metadata as long[] and byte[]. {@code absolute} controls whether
+ * current term is delta encoded according to latest term.
+ * Usually elements in {@code longs} are file pointers, so each one always
+ * increases when a new term is consumed. {@code out} is used to write generic
+ * bytes, which are not monotonic.
+ *
+ * NOTE: sometimes long[] might contain "don't care" values that are unused, e.g.
+ * the pointer to postings list may not be defined for some terms but is defined
+ * for others, if it is designed to inline some postings data in term dictionary.
+ * In this case, the postings writer should always use the last value, so that each
+ * element in metadata long[] remains monotonic.
+ */
+ public abstract void encodeTerm(long[] longs, DataOutput out, FieldInfo fieldInfo, BlockTermState state, boolean absolute) throws IOException;
+
+ /**
+ * Sets the current field for writing, and returns the
+ * fixed length of long[] metadata (which is fixed per
+ * field), called when the writing switches to another field. */
+ // TODO: better name?
+ public int setField(FieldInfo fieldInfo) {
+ this.fieldInfo = fieldInfo;
+ indexOptions = fieldInfo.getIndexOptions();
+
+ writeFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ writePositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ writeOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ writePayloads = fieldInfo.hasPayloads();
+
+ if (writeFreqs == false) {
+ enumFlags = 0;
+ } else if (writePositions == false) {
+ enumFlags = DocsEnum.FLAG_FREQS;
+ } else if (writeOffsets == false) {
+ if (writePayloads) {
+ enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS;
+ } else {
+ enumFlags = 0;
+ }
+ } else {
+ if (writePayloads) {
+ enumFlags = DocsAndPositionsEnum.FLAG_PAYLOADS | DocsAndPositionsEnum.FLAG_OFFSETS;
+ } else {
+ enumFlags = DocsAndPositionsEnum.FLAG_OFFSETS;
+ }
+ }
+
+ return 0;
+ }
+
+ @Override
+ public final BlockTermState writeTerm(BytesRef term, TermsEnum termsEnum, FixedBitSet docsSeen) throws IOException {
+ startTerm();
+ if (writePositions == false) {
+ docsEnum = termsEnum.docs(null, docsEnum, enumFlags);
+ } else {
+ posEnum = termsEnum.docsAndPositions(null, posEnum, enumFlags);
+ docsEnum = posEnum;
+ }
+ assert docsEnum != null;
+
+ int docFreq = 0;
+ long totalTermFreq = 0;
+ while (true) {
+ int docID = docsEnum.nextDoc();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ docFreq++;
+ docsSeen.set(docID);
+ int freq;
+ if (writeFreqs) {
+ freq = docsEnum.freq();
+ totalTermFreq += freq;
+ } else {
+ freq = -1;
+ }
+ startDoc(docID, freq);
+
+ if (writePositions) {
+ for(int i=0;i<freq;i++) {
+ int pos = posEnum.nextPosition();
+ BytesRef payload = writePayloads ? posEnum.getPayload() : null;
+ int startOffset;
+ int endOffset;
+ if (writeOffsets) {
+ startOffset = posEnum.startOffset();
+ endOffset = posEnum.endOffset();
+ } else {
+ startOffset = -1;
+ endOffset = -1;
+ }
+ addPosition(pos, payload, startOffset, endOffset);
+ }
+ }
+
+ finishDoc();
+ }
+
+ if (docFreq == 0) {
+ return null;
+ } else {
+ BlockTermState state = newTermState();
+ state.docFreq = docFreq;
+ state.totalTermFreq = writeFreqs ? totalTermFreq : -1;
+ finishTerm(state);
+ return state;
+ }
+ }
+
+ /** Adds a new doc in this term.
+ * <code>freq</code> will be -1 when term frequencies are omitted
+ * for the field. */
+ public abstract void startDoc(int docID, int freq) throws IOException;
+
+ /** Add a new position & payload, and start/end offset. A
+ * null payload means no payload; a non-null payload with
+ * zero length also means no payload. Caller may reuse
+ * the {@link BytesRef} for the payload between calls
+ * (method must fully consume the payload). <code>startOffset</code>
+ * and <code>endOffset</code> will be -1 when offsets are not indexed. */
+ public abstract void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException;
+
+ /** Called when we are done adding positions & payloads
+ * for each doc. */
+ public abstract void finishDoc() throws IOException;
+}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/blocktree/BlockTreeTermsWriter.java Mon Aug 25 01:52:08 2014
@@ -25,10 +25,7 @@ import java.util.List;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.PostingsConsumer;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.TermStats;
-import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
@@ -252,7 +249,6 @@ public final class BlockTreeTermsWriter
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
- FieldInfo currentField;
private static class FieldMetaData {
public final FieldInfo fieldInfo;
@@ -329,7 +325,6 @@ public final class BlockTreeTermsWriter
indexOut = state.directory.createOutput(termsIndexFileName, state.context);
writeIndexHeader(indexOut);
- currentField = null;
this.postingsWriter = postingsWriter;
// segment = state.segmentInfo.name;
@@ -365,14 +360,34 @@ public final class BlockTreeTermsWriter
indexOut.writeLong(dirStart);
}
- public TermsConsumer addField(FieldInfo field) throws IOException {
- //DEBUG = field.name.equals("id");
- //if (DEBUG) System.out.println("\nBTTW.addField seg=" + segment + " field=" + field.name);
- assert currentField == null || currentField.name.compareTo(field.name) < 0;
- currentField = field;
- return new TermsWriter(field);
- }
+ @Override
+ public void write(Fields fields) throws IOException {
+
+ String lastField = null;
+ for(String field : fields) {
+ assert lastField == null || lastField.compareTo(field) < 0;
+ lastField = field;
+
+ Terms terms = fields.terms(field);
+ if (terms == null) {
+ continue;
+ }
+
+ TermsEnum termsEnum = terms.iterator(null);
+
+ TermsWriter termsWriter = new TermsWriter(fieldInfos.fieldInfo(field));
+ while (true) {
+ BytesRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ termsWriter.write(term, termsEnum);
+ }
+ termsWriter.finish();
+ }
+ }
+
static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
assert fp < (1L << 62);
return (fp << 2) | (hasTerms ? OUTPUT_FLAG_HAS_TERMS : 0) | (isFloor ? OUTPUT_FLAG_IS_FLOOR : 0);
@@ -527,14 +542,13 @@ public final class BlockTreeTermsWriter
private final RAMOutputStream scratchBytes = new RAMOutputStream();
private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
- class TermsWriter extends TermsConsumer {
+ class TermsWriter {
private final FieldInfo fieldInfo;
private final int longsSize;
private long numTerms;
final FixedBitSet docsSeen;
long sumTotalTermFreq;
long sumDocFreq;
- int docCount;
long indexStartFP;
// Records index into pending where the current prefix at that
@@ -868,27 +882,8 @@ public final class BlockTreeTermsWriter
this.longs = new long[longsSize];
}
- @Override
- public Comparator<BytesRef> getComparator() {
- return BytesRef.getUTF8SortedAsUnicodeComparator();
- }
-
- @Override
- public PostingsConsumer startTerm(BytesRef text) throws IOException {
- //if (DEBUG) System.out.println("\nBTTW.startTerm term=" + fieldInfo.name + ":" + toString(text) + " seg=" + segment);
- postingsWriter.startTerm();
- /*
- if (fieldInfo.name.equals("id")) {
- postingsWriter.termID = Integer.parseInt(text.utf8ToString());
- } else {
- postingsWriter.termID = -1;
- }
- */
- return postingsWriter;
- }
-
- @Override
- public void finishTerm(BytesRef text, TermStats stats) throws IOException {
+ /** Writes one term's worth of postings. */
+ public void write(BytesRef text, TermsEnum termsEnum) throws IOException {
/*
if (DEBUG) {
int[] tmp = new int[lastTerm.length];
@@ -897,27 +892,22 @@ public final class BlockTreeTermsWriter
}
*/
- assert stats.docFreq > 0;
- //if (DEBUG) System.out.println("BTTW.finishTerm term=" + fieldInfo.name + ":" + toString(text) + " seg=" + segment + " df=" + stats.docFreq);
-
- assert fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY || stats.totalTermFreq >= stats.docFreq: "postingsWriter=" + postingsWriter;
-
- BlockTermState state = postingsWriter.newTermState();
- state.docFreq = stats.docFreq;
- state.totalTermFreq = stats.totalTermFreq;
- postingsWriter.finishTerm(state);
-
- sumDocFreq += state.docFreq;
- sumTotalTermFreq += state.totalTermFreq;
- pushTerm(text);
+ BlockTermState state = postingsWriter.writeTerm(text, termsEnum, docsSeen);
+ if (state != null) {
+ assert state.docFreq != 0;
+ assert fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY || state.totalTermFreq >= state.docFreq: "postingsWriter=" + postingsWriter;
+ sumDocFreq += state.docFreq;
+ sumTotalTermFreq += state.totalTermFreq;
+ pushTerm(text);
- PendingTerm term = new PendingTerm(text, state);
- pending.add(term);
- numTerms++;
- if (firstPendingTerm == null) {
- firstPendingTerm = term;
+ PendingTerm term = new PendingTerm(text, state);
+ pending.add(term);
+ numTerms++;
+ if (firstPendingTerm == null) {
+ firstPendingTerm = term;
+ }
+ lastPendingTerm = term;
}
- lastPendingTerm = term;
}
/** Pushes the new term to the top of the stack, and writes new blocks. */
@@ -957,8 +947,8 @@ public final class BlockTreeTermsWriter
lastTerm.copyBytes(text);
}
- @Override
- public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException {
+ // Finishes all terms in this field
+ public void finish() throws IOException {
if (numTerms > 0) {
// if (DEBUG) System.out.println("BTTW: finish prefixStarts=" + Arrays.toString(prefixStarts));
@@ -976,10 +966,6 @@ public final class BlockTreeTermsWriter
assert root.prefix.length == 0;
assert root.index.getEmptyOutput() != null;
- this.sumTotalTermFreq = sumTotalTermFreq;
- this.sumDocFreq = sumDocFreq;
- this.docCount = docCount;
-
// Write FST to index
indexStartFP = indexOut.getFilePointer();
root.index.save(indexOut);
@@ -1006,13 +992,13 @@ public final class BlockTreeTermsWriter
indexStartFP,
sumTotalTermFreq,
sumDocFreq,
- docCount,
+ docsSeen.cardinality(),
longsSize,
minTerm, maxTerm));
} else {
assert sumTotalTermFreq == 0 || fieldInfo.getIndexOptions() == IndexOptions.DOCS_ONLY && sumTotalTermFreq == -1;
assert sumDocFreq == 0;
- assert docCount == 0;
+ assert docsSeen.cardinality() == 0;
}
}
@@ -1068,4 +1054,9 @@ public final class BlockTreeTermsWriter
out.writeVInt(bytes.length);
out.writeBytes(bytes.bytes, bytes.offset, bytes.length);
}
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/lucene41/Lucene41PostingsWriter.java Mon Aug 25 01:52:08 2014
@@ -17,31 +17,27 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
-import static org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene41.ForUtil.MAX_DATA_SIZE;
-import static org.apache.lucene.codecs.lucene41.ForUtil.MAX_ENCODED_SIZE;
-
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
-import org.apache.lucene.codecs.PostingsWriterBase;
+import org.apache.lucene.codecs.PushPostingsWriterBase;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.TermState;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.RAMOutputStream;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
+import static org.apache.lucene.codecs.lucene41.ForUtil.MAX_DATA_SIZE;
+import static org.apache.lucene.codecs.lucene41.ForUtil.MAX_ENCODED_SIZE;
+import static org.apache.lucene.codecs.lucene41.Lucene41PostingsFormat.BLOCK_SIZE;
+
/**
* Concrete class that writes docId(maybe frq,pos,offset,payloads) list
@@ -52,7 +48,7 @@ import org.apache.lucene.util.packed.Pac
* @see Lucene41SkipWriter for details about skipping setting and postings layout.
* @lucene.experimental
*/
-public final class Lucene41PostingsWriter extends PostingsWriterBase {
+public final class Lucene41PostingsWriter extends PushPostingsWriterBase {
/**
* Expert: The maximum number of skip levels. Smaller values result in
@@ -78,12 +74,6 @@ public final class Lucene41PostingsWrite
final static IntBlockTermState emptyState = new IntBlockTermState();
IntBlockTermState lastState;
- // How current field indexes postings:
- private boolean fieldHasFreqs;
- private boolean fieldHasPositions;
- private boolean fieldHasOffsets;
- private boolean fieldHasPayloads;
-
// Holds starting file pointers for current term:
private long docStartFP;
private long posStartFP;
@@ -242,15 +232,11 @@ public final class Lucene41PostingsWrite
@Override
public int setField(FieldInfo fieldInfo) {
- IndexOptions indexOptions = fieldInfo.getIndexOptions();
- fieldHasFreqs = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- fieldHasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- fieldHasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
- fieldHasPayloads = fieldInfo.hasPayloads();
- skipWriter.setField(fieldHasPositions, fieldHasOffsets, fieldHasPayloads);
+ super.setField(fieldInfo);
+ skipWriter.setField(writePositions, writeOffsets, writePayloads);
lastState = emptyState;
- if (fieldHasPositions) {
- if (fieldHasPayloads || fieldHasOffsets) {
+ if (writePositions) {
+ if (writePayloads || writeOffsets) {
return 3; // doc + pos + pay FP
} else {
return 2; // doc + pos FP
@@ -263,9 +249,9 @@ public final class Lucene41PostingsWrite
@Override
public void startTerm() {
docStartFP = docOut.getFilePointer();
- if (fieldHasPositions) {
+ if (writePositions) {
posStartFP = posOut.getFilePointer();
- if (fieldHasPayloads || fieldHasOffsets) {
+ if (writePayloads || writeOffsets) {
payStartFP = payOut.getFilePointer();
}
}
@@ -302,7 +288,7 @@ public final class Lucene41PostingsWrite
// if (DEBUG) {
// System.out.println(" docDeltaBuffer[" + docBufferUpto + "]=" + docDelta);
// }
- if (fieldHasFreqs) {
+ if (writeFreqs) {
freqBuffer[docBufferUpto] = termDocFreq;
}
docBufferUpto++;
@@ -313,7 +299,7 @@ public final class Lucene41PostingsWrite
// System.out.println(" write docDelta block @ fp=" + docOut.getFilePointer());
// }
forUtil.writeBlock(docDeltaBuffer, encoded, docOut);
- if (fieldHasFreqs) {
+ if (writeFreqs) {
// if (DEBUG) {
// System.out.println(" write freq block @ fp=" + docOut.getFilePointer());
// }
@@ -330,14 +316,13 @@ public final class Lucene41PostingsWrite
lastStartOffset = 0;
}
- /** Add a new position & payload */
@Override
public void addPosition(int position, BytesRef payload, int startOffset, int endOffset) throws IOException {
// if (DEBUG) {
- // System.out.println("FPW.addPosition pos=" + position + " posBufferUpto=" + posBufferUpto + (fieldHasPayloads ? " payloadByteUpto=" + payloadByteUpto: ""));
+ // System.out.println("FPW.addPosition pos=" + position + " posBufferUpto=" + posBufferUpto + (writePayloads ? " payloadByteUpto=" + payloadByteUpto: ""));
// }
posDeltaBuffer[posBufferUpto] = position - lastPosition;
- if (fieldHasPayloads) {
+ if (writePayloads) {
if (payload == null || payload.length == 0) {
// no payload
payloadLengthBuffer[posBufferUpto] = 0;
@@ -351,7 +336,7 @@ public final class Lucene41PostingsWrite
}
}
- if (fieldHasOffsets) {
+ if (writeOffsets) {
assert startOffset >= lastStartOffset;
assert endOffset >= startOffset;
offsetStartDeltaBuffer[posBufferUpto] = startOffset - lastStartOffset;
@@ -367,13 +352,13 @@ public final class Lucene41PostingsWrite
// }
forUtil.writeBlock(posDeltaBuffer, encoded, posOut);
- if (fieldHasPayloads) {
+ if (writePayloads) {
forUtil.writeBlock(payloadLengthBuffer, encoded, payOut);
payOut.writeVInt(payloadByteUpto);
payOut.writeBytes(payloadBytes, 0, payloadByteUpto);
payloadByteUpto = 0;
}
- if (fieldHasOffsets) {
+ if (writeOffsets) {
forUtil.writeBlock(offsetStartDeltaBuffer, encoded, payOut);
forUtil.writeBlock(offsetLengthBuffer, encoded, payOut);
}
@@ -434,7 +419,7 @@ public final class Lucene41PostingsWrite
for(int i=0;i<docBufferUpto;i++) {
final int docDelta = docDeltaBuffer[i];
final int freq = freqBuffer[i];
- if (!fieldHasFreqs) {
+ if (!writeFreqs) {
docOut.writeVInt(docDelta);
} else if (freqBuffer[i] == 1) {
docOut.writeVInt((docDelta<<1)|1);
@@ -447,10 +432,10 @@ public final class Lucene41PostingsWrite
final long lastPosBlockOffset;
- if (fieldHasPositions) {
+ if (writePositions) {
// if (DEBUG) {
// if (posBufferUpto > 0) {
- // System.out.println(" write pos vInt block (count=" + posBufferUpto + ") at fp=" + posOut.getFilePointer() + " posStartFP=" + posStartFP + " hasPayloads=" + fieldHasPayloads + " hasOffsets=" + fieldHasOffsets);
+ // System.out.println(" write pos vInt block (count=" + posBufferUpto + ") at fp=" + posOut.getFilePointer() + " posStartFP=" + posStartFP + " hasPayloads=" + writePayloads + " hasOffsets=" + writeOffsets);
// }
// }
@@ -475,7 +460,7 @@ public final class Lucene41PostingsWrite
int payloadBytesReadUpto = 0;
for(int i=0;i<posBufferUpto;i++) {
final int posDelta = posDeltaBuffer[i];
- if (fieldHasPayloads) {
+ if (writePayloads) {
final int payloadLength = payloadLengthBuffer[i];
if (payloadLength != lastPayloadLength) {
lastPayloadLength = payloadLength;
@@ -500,7 +485,7 @@ public final class Lucene41PostingsWrite
posOut.writeVInt(posDelta);
}
- if (fieldHasOffsets) {
+ if (writeOffsets) {
// if (DEBUG) {
// System.out.println(" write offset @ pos.fp=" + posOut.getFilePointer());
// }
@@ -516,7 +501,7 @@ public final class Lucene41PostingsWrite
}
}
- if (fieldHasPayloads) {
+ if (writePayloads) {
assert payloadBytesReadUpto == payloadByteUpto;
payloadByteUpto = 0;
}
@@ -563,16 +548,16 @@ public final class Lucene41PostingsWrite
lastState = emptyState;
}
longs[0] = state.docStartFP - lastState.docStartFP;
- if (fieldHasPositions) {
+ if (writePositions) {
longs[1] = state.posStartFP - lastState.posStartFP;
- if (fieldHasPayloads || fieldHasOffsets) {
+ if (writePayloads || writeOffsets) {
longs[2] = state.payStartFP - lastState.payStartFP;
}
}
if (state.singletonDocID != -1) {
out.writeVInt(state.singletonDocID);
}
- if (fieldHasPositions) {
+ if (writePositions) {
if (state.lastPosBlockOffset != -1) {
out.writeVLong(state.lastPosBlockOffset);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/codecs/perfield/PerFieldPostingsFormat.java Mon Aug 25 01:52:08 2014
@@ -19,24 +19,32 @@ package org.apache.lucene.codecs.perfiel
import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.ServiceLoader; // javadocs
+import java.util.Set;
import java.util.TreeMap;
+import java.util.TreeSet;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Fields;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
+import static org.apache.lucene.index.FilterAtomicReader.FilterFields;
+
/**
* Enables per field postings support.
* <p>
@@ -65,108 +73,136 @@ public abstract class PerFieldPostingsFo
* segment suffix name for each field. */
public static final String PER_FIELD_SUFFIX_KEY = PerFieldPostingsFormat.class.getSimpleName() + ".suffix";
-
/** Sole constructor. */
public PerFieldPostingsFormat() {
super(PER_FIELD_NAME);
}
- @Override
- public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
- throws IOException {
- return new FieldsWriter(state);
- }
-
- static class FieldsConsumerAndSuffix implements Closeable {
- FieldsConsumer consumer;
+ /** Group of fields written by one PostingsFormat */
+ static class FieldsGroup {
+ final Set<String> fields = new TreeSet<>();
int suffix;
-
- @Override
- public void close() throws IOException {
- consumer.close();
+
+ /** Custom SegmentWriteState for this group of fields,
+ * with the segmentSuffix uniqueified for this
+ * PostingsFormat */
+ SegmentWriteState state;
+ };
+
+ static String getSuffix(String formatName, String suffix) {
+ return formatName + "_" + suffix;
+ }
+
+ static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) {
+ if (outerSegmentSuffix.length() == 0) {
+ return segmentSuffix;
+ } else {
+ // TODO: support embedding; I think it should work but
+ // we need a test confirm to confirm
+ // return outerSegmentSuffix + "_" + segmentSuffix;
+ throw new IllegalStateException("cannot embed PerFieldPostingsFormat inside itself (field \"" + fieldName + "\" returned PerFieldPostingsFormat)");
}
}
-
+
private class FieldsWriter extends FieldsConsumer {
+ final SegmentWriteState writeState;
+ final List<Closeable> toClose = new ArrayList<Closeable>();
- private final Map<PostingsFormat,FieldsConsumerAndSuffix> formats = new HashMap<>();
- private final Map<String,Integer> suffixes = new HashMap<>();
-
- private final SegmentWriteState segmentWriteState;
-
- public FieldsWriter(SegmentWriteState state) {
- segmentWriteState = state;
+ public FieldsWriter(SegmentWriteState writeState) {
+ this.writeState = writeState;
}
@Override
- public TermsConsumer addField(FieldInfo field) throws IOException {
- final PostingsFormat format = getPostingsFormatForField(field.name);
- if (format == null) {
- throw new IllegalStateException("invalid null PostingsFormat for field=\"" + field.name + "\"");
- }
- final String formatName = format.getName();
-
- String previousValue = field.putAttribute(PER_FIELD_FORMAT_KEY, formatName);
- assert previousValue == null;
-
- Integer suffix;
+ public void write(Fields fields) throws IOException {
+
+ // Maps a PostingsFormat instance to the suffix it
+ // should use
+ Map<PostingsFormat,FieldsGroup> formatToGroups = new HashMap<>();
+
+ // Holds last suffix of each PostingFormat name
+ Map<String,Integer> suffixes = new HashMap<>();
+
+ // First pass: assign field -> PostingsFormat
+ for(String field : fields) {
+ FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(field);
+
+ final PostingsFormat format = getPostingsFormatForField(field);
+
+ if (format == null) {
+ throw new IllegalStateException("invalid null PostingsFormat for field=\"" + field + "\"");
+ }
+ String formatName = format.getName();
- FieldsConsumerAndSuffix consumer = formats.get(format);
- if (consumer == null) {
- // First time we are seeing this format; create a new instance
-
- // bump the suffix
- suffix = suffixes.get(formatName);
- if (suffix == null) {
- suffix = 0;
+ FieldsGroup group = formatToGroups.get(format);
+ if (group == null) {
+ // First time we are seeing this format; create a
+ // new instance
+
+ // bump the suffix
+ Integer suffix = suffixes.get(formatName);
+ if (suffix == null) {
+ suffix = 0;
+ } else {
+ suffix = suffix + 1;
+ }
+ suffixes.put(formatName, suffix);
+
+ String segmentSuffix = getFullSegmentSuffix(field,
+ writeState.segmentSuffix,
+ getSuffix(formatName, Integer.toString(suffix)));
+ group = new FieldsGroup();
+ group.state = new SegmentWriteState(writeState, segmentSuffix);
+ group.suffix = suffix;
+ formatToGroups.put(format, group);
} else {
- suffix = suffix + 1;
+ // we've already seen this format, so just grab its suffix
+ assert suffixes.containsKey(formatName);
}
- suffixes.put(formatName, suffix);
-
- final String segmentSuffix = getFullSegmentSuffix(field.name,
- segmentWriteState.segmentSuffix,
- getSuffix(formatName, Integer.toString(suffix)));
- consumer = new FieldsConsumerAndSuffix();
- consumer.consumer = format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix));
- consumer.suffix = suffix;
- formats.put(format, consumer);
- } else {
- // we've already seen this format, so just grab its suffix
- assert suffixes.containsKey(formatName);
- suffix = consumer.suffix;
+
+ group.fields.add(field);
+
+ String previousValue = fieldInfo.putAttribute(PER_FIELD_FORMAT_KEY, formatName);
+ assert previousValue == null;
+
+ previousValue = fieldInfo.putAttribute(PER_FIELD_SUFFIX_KEY, Integer.toString(group.suffix));
+ assert previousValue == null;
}
-
- previousValue = field.putAttribute(PER_FIELD_SUFFIX_KEY, Integer.toString(suffix));
- assert previousValue == null;
- // TODO: we should only provide the "slice" of FIS
- // that this PF actually sees ... then stuff like
- // .hasProx could work correctly?
- // NOTE: .hasProx is already broken in the same way for the non-perfield case,
- // if there is a fieldinfo with prox that has no postings, you get a 0 byte file.
- return consumer.consumer.addField(field);
+ // Second pass: write postings
+ boolean success = false;
+ try {
+ for(Map.Entry<PostingsFormat,FieldsGroup> ent : formatToGroups.entrySet()) {
+ PostingsFormat format = ent.getKey();
+ final FieldsGroup group = ent.getValue();
+
+ // Exposes only the fields from this group:
+ Fields maskedFields = new FilterFields(fields) {
+ @Override
+ public Iterator<String> iterator() {
+ return group.fields.iterator();
+ }
+ };
+
+ FieldsConsumer consumer = format.fieldsConsumer(group.state);
+ toClose.add(consumer);
+ consumer.write(maskedFields);
+ }
+ success = true;
+ } finally {
+ if (success == false) {
+ IOUtils.closeWhileHandlingException(toClose);
+ }
+ }
}
@Override
- public void close() throws IOException {
- // Close all subs
- IOUtils.close(formats.values());
+ public Comparator<BytesRef> getComparator() {
+ return BytesRef.getUTF8SortedAsUnicodeComparator();
}
- }
-
- static String getSuffix(String formatName, String suffix) {
- return formatName + "_" + suffix;
- }
- static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) {
- if (outerSegmentSuffix.length() == 0) {
- return segmentSuffix;
- } else {
- // TODO: support embedding; I think it should work but
- // we need a test confirm to confirm
- // return outerSegmentSuffix + "_" + segmentSuffix;
- throw new IllegalStateException("cannot embed PerFieldPostingsFormat inside itself (field \"" + fieldName + "\" returned PerFieldPostingsFormat)");
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(toClose);
}
}
@@ -249,6 +285,12 @@ public abstract class PerFieldPostingsFo
}
@Override
+ public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
+ throws IOException {
+ return new FieldsWriter(state);
+ }
+
+ @Override
public final FieldsProducer fieldsProducer(SegmentReadState state)
throws IOException {
return new FieldsReader(state);
Added: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java?rev=1620250&view=auto
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (added)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java Mon Aug 25 01:52:08 2014
@@ -0,0 +1,546 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
+import org.apache.lucene.util.AttributeSource; // javadocs
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+
+/** Implements limited (iterators only, no stats) {@link
+ * Fields} interface over the in-RAM buffered
+ * fields/terms/postings, to flush postings through the
+ * PostingsFormat. */
+
+class FreqProxFields extends Fields {
+ final Map<String,FreqProxTermsWriterPerField> fields = new LinkedHashMap<>();
+ final Comparator<BytesRef> comparator;
+
+ public FreqProxFields(List<FreqProxTermsWriterPerField> fieldList, Comparator<BytesRef> comparator) {
+ // NOTE: fields are already sorted by field name
+ for(FreqProxTermsWriterPerField field : fieldList) {
+ fields.put(field.fieldInfo.name, field);
+ }
+ this.comparator = comparator;
+ }
+
+ public Iterator<String> iterator() {
+ return fields.keySet().iterator();
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ FreqProxTermsWriterPerField perField = fields.get(field);
+ return perField == null ? null : new FreqProxTerms(perField, comparator);
+ }
+
+ @Override
+ public int size() {
+ //return fields.size();
+ throw new UnsupportedOperationException();
+ }
+
+ private static class FreqProxTerms extends Terms {
+ final FreqProxTermsWriterPerField terms;
+ final Comparator<BytesRef> comparator;
+
+ public FreqProxTerms(FreqProxTermsWriterPerField terms, Comparator<BytesRef> comparator) {
+ this.terms = terms;
+ this.comparator = comparator;
+ }
+
+ @Override
+ public TermsEnum iterator(TermsEnum reuse) {
+ FreqProxTermsEnum termsEnum;
+ if (reuse instanceof FreqProxTermsEnum && ((FreqProxTermsEnum) reuse).terms == this.terms) {
+ termsEnum = (FreqProxTermsEnum) reuse;
+ } else {
+ termsEnum = new FreqProxTermsEnum(terms, comparator);
+ }
+ termsEnum.reset();
+ return termsEnum;
+ }
+
+ @Override
+ public long size() {
+ //return terms.termsHashPerField.bytesHash.size();
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSumTotalTermFreq() {
+ //return terms.sumTotalTermFreq;
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSumDocFreq() {
+ //return terms.sumDocFreq;
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getDocCount() {
+ //return terms.docCount;
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean hasFreqs() {
+ return terms.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ }
+
+ @Override
+ public boolean hasOffsets() {
+ // NOTE: the in-memory buffer may have indexed offsets
+ // because that's what FieldInfo said when we started,
+ // but during indexing this may have been downgraded:
+ return terms.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ }
+
+ @Override
+ public boolean hasPositions() {
+ // NOTE: the in-memory buffer may have indexed positions
+ // because that's what FieldInfo said when we started,
+ // but during indexing this may have been downgraded:
+ return terms.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ }
+
+ @Override
+ public boolean hasPayloads() {
+ return terms.sawPayloads;
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return comparator;
+ }
+ }
+
+ private static class FreqProxTermsEnum extends TermsEnum {
+ final FreqProxTermsWriterPerField terms;
+ final int[] sortedTermIDs;
+ final FreqProxPostingsArray postingsArray;
+ final BytesRef scratch = new BytesRef();
+ final int numTerms;
+ final Comparator<BytesRef> comparator;
+ int ord;
+
+ public FreqProxTermsEnum(FreqProxTermsWriterPerField terms, Comparator<BytesRef> comparator) {
+ this.terms = terms;
+ this.numTerms = terms.bytesHash.size();
+ sortedTermIDs = terms.sortedTermIDs;
+ assert sortedTermIDs != null;
+ postingsArray = (FreqProxPostingsArray) terms.postingsArray;
+ this.comparator = comparator;
+ }
+
+ public void reset() {
+ ord = -1;
+ }
+
+ public SeekStatus seekCeil(BytesRef text) {
+
+ // TODO: we could instead keep the BytesRefHash
+ // intact so this is a hash lookup
+
+ // binary search:
+ int lo = 0;
+ int hi = numTerms - 1;
+ while (hi >= lo) {
+ int mid = (lo + hi) >>> 1;
+ int textStart = postingsArray.textStarts[sortedTermIDs[mid]];
+ terms.bytePool.setBytesRef(scratch, textStart);
+ int cmp = getComparator().compare(scratch, text);
+ if (cmp < 0) {
+ lo = mid + 1;
+ } else if (cmp > 0) {
+ hi = mid - 1;
+ } else {
+ // found:
+ ord = mid;
+ return SeekStatus.FOUND;
+ }
+ }
+
+ // not found:
+ ord = lo + 1;
+ if (ord >= numTerms) {
+ return SeekStatus.END;
+ } else {
+ int textStart = postingsArray.textStarts[sortedTermIDs[ord]];
+ terms.bytePool.setBytesRef(scratch, textStart);
+ return SeekStatus.NOT_FOUND;
+ }
+ }
+
+ public void seekExact(long ord) {
+ this.ord = (int) ord;
+ int textStart = postingsArray.textStarts[sortedTermIDs[this.ord]];
+ terms.bytePool.setBytesRef(scratch, textStart);
+ }
+
+ @Override
+ public BytesRef next() {
+ ord++;
+ if (ord >= numTerms) {
+ return null;
+ } else {
+ int textStart = postingsArray.textStarts[sortedTermIDs[ord]];
+ terms.bytePool.setBytesRef(scratch, textStart);
+ return scratch;
+ }
+ }
+
+ @Override
+ public BytesRef term() {
+ return scratch;
+ }
+
+ @Override
+ public long ord() {
+ return ord;
+ }
+
+ @Override
+ public int docFreq() {
+ // We do not store this per-term, and we cannot
+ // implement this at merge time w/o an added pass
+ // through the postings:
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long totalTermFreq() {
+ // We do not store this per-term, and we cannot
+ // implement this at merge time w/o an added pass
+ // through the postings:
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
+ if (liveDocs != null) {
+ throw new IllegalArgumentException("liveDocs must be null");
+ }
+
+ FreqProxDocsEnum docsEnum;
+
+ if (!terms.hasFreq && (flags & DocsEnum.FLAG_FREQS) != 0) {
+ // Caller wants freqs but we didn't index them;
+ // don't lie:
+ throw new IllegalArgumentException("did not index freq");
+ }
+
+ if (reuse instanceof FreqProxDocsEnum) {
+ docsEnum = (FreqProxDocsEnum) reuse;
+ if (docsEnum.postingsArray != postingsArray) {
+ docsEnum = new FreqProxDocsEnum(terms, postingsArray);
+ }
+ } else {
+ docsEnum = new FreqProxDocsEnum(terms, postingsArray);
+ }
+ docsEnum.reset(sortedTermIDs[ord]);
+ return docsEnum;
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
+ if (liveDocs != null) {
+ throw new IllegalArgumentException("liveDocs must be null");
+ }
+ FreqProxDocsAndPositionsEnum posEnum;
+
+ if (!terms.hasProx) {
+ // Caller wants positions but we didn't index them;
+ // don't lie:
+ throw new IllegalArgumentException("did not index positions");
+ }
+
+ if (!terms.hasOffsets && (flags & DocsAndPositionsEnum.FLAG_OFFSETS) != 0) {
+ // Caller wants offsets but we didn't index them;
+ // don't lie:
+ throw new IllegalArgumentException("did not index offsets");
+ }
+
+ if (reuse instanceof FreqProxDocsAndPositionsEnum) {
+ posEnum = (FreqProxDocsAndPositionsEnum) reuse;
+ if (posEnum.postingsArray != postingsArray) {
+ posEnum = new FreqProxDocsAndPositionsEnum(terms, postingsArray);
+ }
+ } else {
+ posEnum = new FreqProxDocsAndPositionsEnum(terms, postingsArray);
+ }
+ posEnum.reset(sortedTermIDs[ord]);
+ return posEnum;
+ }
+
+ /**
+ * Expert: Returns the TermsEnums internal state to position the TermsEnum
+ * without re-seeking the term dictionary.
+ * <p>
+ * NOTE: A seek by {@link TermState} might not capture the
+ * {@link AttributeSource}'s state. Callers must maintain the
+ * {@link AttributeSource} states separately
+ *
+ * @see TermState
+ * @see #seekExact(BytesRef, TermState)
+ */
+ public TermState termState() throws IOException {
+ return new TermState() {
+ @Override
+ public void copyFrom(TermState other) {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+
+ @Override
+ public Comparator<BytesRef> getComparator() {
+ return comparator;
+ }
+ }
+
+ private static class FreqProxDocsEnum extends DocsEnum {
+
+ final FreqProxTermsWriterPerField terms;
+ final FreqProxPostingsArray postingsArray;
+ final ByteSliceReader reader = new ByteSliceReader();
+ final boolean readTermFreq;
+ int docID;
+ int freq;
+ boolean ended;
+ int termID;
+
+ public FreqProxDocsEnum(FreqProxTermsWriterPerField terms, FreqProxPostingsArray postingsArray) {
+ this.terms = terms;
+ this.postingsArray = postingsArray;
+ this.readTermFreq = terms.hasFreq;
+ }
+
+ public void reset(int termID) {
+ this.termID = termID;
+ terms.initReader(reader, termID, 0);
+ ended = false;
+ docID = 0;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int freq() {
+ // Don't lie here ... don't want codecs writings lots
+ // of wasted 1s into the index:
+ if (!readTermFreq) {
+ throw new IllegalStateException("freq was not indexed");
+ } else {
+ return freq;
+ }
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (reader.eof()) {
+ if (ended) {
+ return NO_MORE_DOCS;
+ } else {
+ ended = true;
+ docID = postingsArray.lastDocIDs[termID];
+ if (readTermFreq) {
+ freq = postingsArray.termFreqs[termID];
+ }
+ }
+ } else {
+ int code = reader.readVInt();
+ if (!readTermFreq) {
+ docID += code;
+ } else {
+ docID += code >>> 1;
+ if ((code & 1) != 0) {
+ freq = 1;
+ } else {
+ freq = reader.readVInt();
+ }
+ }
+
+ assert docID != postingsArray.lastDocIDs[termID];
+ }
+
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long cost() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class FreqProxDocsAndPositionsEnum extends DocsAndPositionsEnum {
+
+ final FreqProxTermsWriterPerField terms;
+ final FreqProxPostingsArray postingsArray;
+ final ByteSliceReader reader = new ByteSliceReader();
+ final ByteSliceReader posReader = new ByteSliceReader();
+ final boolean readOffsets;
+ int docID;
+ int freq;
+ int pos;
+ int startOffset;
+ int endOffset;
+ int posLeft;
+ int termID;
+ boolean ended;
+ boolean hasPayload;
+ BytesRefBuilder payload = new BytesRefBuilder();
+
+ public FreqProxDocsAndPositionsEnum(FreqProxTermsWriterPerField terms, FreqProxPostingsArray postingsArray) {
+ this.terms = terms;
+ this.postingsArray = postingsArray;
+ this.readOffsets = terms.hasOffsets;
+ assert terms.hasProx;
+ assert terms.hasFreq;
+ }
+
+ public void reset(int termID) {
+ this.termID = termID;
+ terms.initReader(reader, termID, 0);
+ terms.initReader(posReader, termID, 1);
+ ended = false;
+ docID = 0;
+ posLeft = 0;
+ }
+
+ @Override
+ public int docID() {
+ return docID;
+ }
+
+ @Override
+ public int freq() {
+ return freq;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ while (posLeft != 0) {
+ nextPosition();
+ }
+
+ if (reader.eof()) {
+ if (ended) {
+ return NO_MORE_DOCS;
+ } else {
+ ended = true;
+ docID = postingsArray.lastDocIDs[termID];
+ freq = postingsArray.termFreqs[termID];
+ }
+ } else {
+ int code = reader.readVInt();
+ docID += code >>> 1;
+ if ((code & 1) != 0) {
+ freq = 1;
+ } else {
+ freq = reader.readVInt();
+ }
+
+ assert docID != postingsArray.lastDocIDs[termID];
+ }
+
+ posLeft = freq;
+ pos = 0;
+ startOffset = 0;
+ return docID;
+ }
+
+ @Override
+ public int advance(int target) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long cost() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ assert posLeft > 0;
+ posLeft--;
+ int code = posReader.readVInt();
+ pos += code >>> 1;
+ if ((code & 1) != 0) {
+ hasPayload = true;
+ // has a payload
+ payload.setLength(posReader.readVInt());
+ payload.grow(payload.length());
+ posReader.readBytes(payload.bytes(), 0, payload.length());
+ } else {
+ hasPayload = false;
+ }
+
+ if (readOffsets) {
+ startOffset += posReader.readVInt();
+ endOffset = startOffset + posReader.readVInt();
+ }
+
+ return pos;
+ }
+
+ @Override
+ public int startOffset() {
+ if (!readOffsets) {
+ throw new IllegalStateException("offsets were not indexed");
+ }
+ return startOffset;
+ }
+
+ @Override
+ public int endOffset() {
+ if (!readOffsets) {
+ throw new IllegalStateException("offsets were not indexed");
+ }
+ return endOffset;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ if (hasPayload) {
+ return payload.get();
+ } else {
+ return null;
+ }
+ }
+ }
+}
\ No newline at end of file
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Mon Aug 25 01:52:08 2014
@@ -19,10 +19,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.Map;
import org.apache.lucene.codecs.FieldsConsumer;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
@@ -32,64 +35,80 @@ final class FreqProxTermsWriter extends
super(docWriter, true, termVectors);
}
- @Override
- public void flush(Map<String,TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
- super.flush(fieldsToFlush, state);
-
- // Gather all fields that saw any postings:
- List<FreqProxTermsWriterPerField> allFields = new ArrayList<>();
+ private void applyDeletes(SegmentWriteState state, Fields fields) throws IOException {
- for (TermsHashPerField f : fieldsToFlush.values()) {
- final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f;
- if (perField.bytesHash.size() > 0) {
- allFields.add(perField);
+ // Process any pending Term deletes for this newly
+ // flushed segment:
+ if (state.segUpdates != null && state.segUpdates.terms.size() > 0) {
+ Map<Term,Integer> segDeletes = state.segUpdates.terms;
+ List<Term> deleteTerms = new ArrayList<>(segDeletes.keySet());
+ Collections.sort(deleteTerms);
+ String lastField = null;
+ TermsEnum termsEnum = null;
+ DocsEnum docsEnum = null;
+ for(Term deleteTerm : deleteTerms) {
+ if (deleteTerm.field().equals(lastField) == false) {
+ lastField = deleteTerm.field();
+ Terms terms = fields.terms(lastField);
+ if (terms != null) {
+ termsEnum = terms.iterator(termsEnum);
+ } else {
+ termsEnum = null;
+ }
+ }
+
+ if (termsEnum != null && termsEnum.seekExact(deleteTerm.bytes())) {
+ docsEnum = termsEnum.docs(null, docsEnum, 0);
+ int delDocLimit = segDeletes.get(deleteTerm);
+ assert delDocLimit < DocsEnum.NO_MORE_DOCS;
+ while (true) {
+ int doc = docsEnum.nextDoc();
+ if (doc < delDocLimit) {
+ if (state.liveDocs == null) {
+ state.liveDocs = state.segmentInfo.getCodec().liveDocsFormat().newLiveDocs(state.segmentInfo.getDocCount());
+ }
+ if (state.liveDocs.get(doc)) {
+ state.delCountOnFlush++;
+ state.liveDocs.clear(doc);
+ }
+ } else {
+ break;
+ }
+ }
+ }
}
}
+ }
- final int numAllFields = allFields.size();
-
- // Sort by field name
- CollectionUtil.introSort(allFields);
-
- final FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state);
+ @Override
+ public void flush(Map<String,TermsHashPerField> fieldsToFlush, final SegmentWriteState state) throws IOException {
+ super.flush(fieldsToFlush, state);
boolean success = false;
+ FieldsConsumer consumer = state.segmentInfo.getCodec().postingsFormat().fieldsConsumer(state);
try {
- TermsHash termsHash = null;
+ Comparator<BytesRef> comparator = consumer.getComparator();
+ // Gather all fields that saw any postings:
+ List<FreqProxTermsWriterPerField> allFields = new ArrayList<>();
- /*
- Current writer chain:
- FieldsConsumer
- -> IMPL: FormatPostingsTermsDictWriter
- -> TermsConsumer
- -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
- -> DocsConsumer
- -> IMPL: FormatPostingsDocsWriter
- -> PositionsConsumer
- -> IMPL: FormatPostingsPositionsWriter
- */
-
- for (int fieldNumber = 0; fieldNumber < numAllFields; fieldNumber++) {
- final FieldInfo fieldInfo = allFields.get(fieldNumber).fieldInfo;
-
- final FreqProxTermsWriterPerField fieldWriter = allFields.get(fieldNumber);
-
- // If this field has postings then add them to the
- // segment
- fieldWriter.flush(fieldInfo.name, consumer, state);
-
- TermsHashPerField perField = fieldWriter;
- assert termsHash == null || termsHash == perField.termsHash;
- termsHash = perField.termsHash;
- int numPostings = perField.bytesHash.size();
- perField.reset();
- fieldWriter.reset();
+ for (TermsHashPerField f : fieldsToFlush.values()) {
+ final FreqProxTermsWriterPerField perField = (FreqProxTermsWriterPerField) f;
+ if (perField.bytesHash.size() > 0) {
+ perField.sortPostings(comparator);
+ assert perField.fieldInfo.isIndexed();
+ allFields.add(perField);
+ }
}
- if (termsHash != null) {
- termsHash.reset();
- }
+ // Sort by field name
+ CollectionUtil.introSort(allFields);
+
+ Fields fields = new FreqProxFields(allFields, comparator);
+
+ applyDeletes(state, fields);
+
+ consumer.write(fields);
success = true;
} finally {
if (success) {
@@ -98,6 +117,7 @@ final class FreqProxTermsWriter extends
IOUtils.closeWhileHandlingException(consumer);
}
}
+
}
@Override
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1620250&r1=1620249&r2=1620250&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Mon Aug 25 01:52:08 2014
@@ -18,19 +18,11 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Comparator;
-import java.util.Map;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.PostingsConsumer;
-import org.apache.lucene.codecs.TermStats;
-import org.apache.lucene.codecs.TermsConsumer;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.RamUsageEstimator;
// TODO: break into separate freq and prox writers as
@@ -45,6 +37,11 @@ final class FreqProxTermsWriterPerField
final boolean hasOffsets;
PayloadAttribute payloadAttribute;
OffsetAttribute offsetAttribute;
+ long sumTotalTermFreq;
+ long sumDocFreq;
+
+ // How many docs have this field:
+ int docCount;
/** Set to true if any token had a payload in the current
* segment. */
@@ -62,6 +59,12 @@ final class FreqProxTermsWriterPerField
@Override
void finish() throws IOException {
super.finish();
+ sumDocFreq += fieldState.uniqueTermCount;
+ sumTotalTermFreq += fieldState.length;
+ if (fieldState.length > 0) {
+ docCount++;
+ }
+
if (sawPayloads) {
fieldInfo.setStorePayloads();
}
@@ -274,234 +277,4 @@ final class FreqProxTermsWriterPerField
return bytes;
}
}
-
- /* Walk through all unique text tokens (Posting
- * instances) found in this field and serialize them
- * into a single RAM segment. */
- void flush(String fieldName, FieldsConsumer consumer, final SegmentWriteState state)
- throws IOException {
-
- BytesRefBuilder payload = null;
-
- if (!fieldInfo.isIndexed()) {
- return; // nothing to flush, don't bother the codec with the unindexed field
- }
-
- final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
- final Comparator<BytesRef> termComp = termsConsumer.getComparator();
-
- // CONFUSING: this.indexOptions holds the index options
- // that were current when we first saw this field. But
- // it's possible this has changed, eg when other
- // documents are indexed that cause a "downgrade" of the
- // IndexOptions. So we must decode the in-RAM buffer
- // according to this.indexOptions, but then write the
- // new segment to the directory according to
- // currentFieldIndexOptions:
- final IndexOptions currentFieldIndexOptions = fieldInfo.getIndexOptions();
- assert currentFieldIndexOptions != null;
-
- final boolean writeTermFreq = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
- final boolean writePositions = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
- final boolean writeOffsets = currentFieldIndexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
-
- final boolean readTermFreq = this.hasFreq;
- final boolean readPositions = this.hasProx;
- final boolean readOffsets = this.hasOffsets;
-
- //System.out.println("flush readTF=" + readTermFreq + " readPos=" + readPositions + " readOffs=" + readOffsets);
-
- // Make sure FieldInfo.update is working correctly!:
- assert !writeTermFreq || readTermFreq;
- assert !writePositions || readPositions;
- assert !writeOffsets || readOffsets;
-
- assert !writeOffsets || writePositions;
-
- final Map<Term,Integer> segDeletes;
- if (state.segUpdates != null && state.segUpdates.terms.size() > 0) {
- segDeletes = state.segUpdates.terms;
- } else {
- segDeletes = null;
- }
-
- final int[] termIDs = sortPostings(termComp);
- final int numTerms = bytesHash.size();
- final BytesRef text = new BytesRef();
- final FreqProxPostingsArray postings = freqProxPostingsArray;
- final ByteSliceReader freq = new ByteSliceReader();
- final ByteSliceReader prox = new ByteSliceReader();
-
- FixedBitSet visitedDocs = new FixedBitSet(state.segmentInfo.getDocCount());
- long sumTotalTermFreq = 0;
- long sumDocFreq = 0;
-
- Term protoTerm = new Term(fieldName);
- for (int i = 0; i < numTerms; i++) {
- final int termID = termIDs[i];
- //System.out.println("term=" + termID);
- // Get BytesRef
- final int textStart = postings.textStarts[termID];
- bytePool.setBytesRef(text, textStart);
-
- initReader(freq, termID, 0);
- if (readPositions || readOffsets) {
- initReader(prox, termID, 1);
- }
-
- // TODO: really TermsHashPerField should take over most
- // of this loop, including merge sort of terms from
- // multiple threads and interacting with the
- // TermsConsumer, only calling out to us (passing us the
- // DocsConsumer) to handle delivery of docs/positions
-
- final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
-
- final int delDocLimit;
- if (segDeletes != null) {
- protoTerm.bytes = text;
- final Integer docIDUpto = segDeletes.get(protoTerm);
- if (docIDUpto != null) {
- delDocLimit = docIDUpto;
- } else {
- delDocLimit = 0;
- }
- } else {
- delDocLimit = 0;
- }
-
- // Now termStates has numToMerge FieldMergeStates
- // which all share the same term. Now we must
- // interleave the docID streams.
- int docFreq = 0;
- long totalTermFreq = 0;
- int docID = 0;
-
- while(true) {
- //System.out.println(" cycle");
- final int termFreq;
- if (freq.eof()) {
- if (postings.lastDocCodes[termID] != -1) {
- // Return last doc
- docID = postings.lastDocIDs[termID];
- if (readTermFreq) {
- termFreq = postings.termFreqs[termID];
- } else {
- termFreq = -1;
- }
- postings.lastDocCodes[termID] = -1;
- } else {
- // EOF
- break;
- }
- } else {
- final int code = freq.readVInt();
- if (!readTermFreq) {
- docID += code;
- termFreq = -1;
- } else {
- docID += code >>> 1;
- if ((code & 1) != 0) {
- termFreq = 1;
- } else {
- termFreq = freq.readVInt();
- }
- }
-
- assert docID != postings.lastDocIDs[termID];
- }
-
- docFreq++;
- assert docID < state.segmentInfo.getDocCount(): "doc=" + docID + " maxDoc=" + state.segmentInfo.getDocCount();
-
- // NOTE: we could check here if the docID was
- // deleted, and skip it. However, this is somewhat
- // dangerous because it can yield non-deterministic
- // behavior since we may see the docID before we see
- // the term that caused it to be deleted. This
- // would mean some (but not all) of its postings may
- // make it into the index, which'd alter the docFreq
- // for those terms. We could fix this by doing two
- // passes, ie first sweep marks all del docs, and
- // 2nd sweep does the real flush, but I suspect
- // that'd add too much time to flush.
- visitedDocs.set(docID);
- postingsConsumer.startDoc(docID, writeTermFreq ? termFreq : -1);
- if (docID < delDocLimit) {
- // Mark it deleted. TODO: we could also skip
- // writing its postings; this would be
- // deterministic (just for this Term's docs).
-
- // TODO: can we do this reach-around in a cleaner way????
- if (state.liveDocs == null) {
- state.liveDocs = docState.docWriter.codec.liveDocsFormat().newLiveDocs(state.segmentInfo.getDocCount());
- }
- if (state.liveDocs.get(docID)) {
- state.delCountOnFlush++;
- state.liveDocs.clear(docID);
- }
- }
-
- totalTermFreq += termFreq;
-
- // Carefully copy over the prox + payload info,
- // changing the format to match Lucene's segment
- // format.
-
- if (readPositions || readOffsets) {
- // we did record positions (& maybe payload) and/or offsets
- int position = 0;
- int offset = 0;
- for(int j=0;j<termFreq;j++) {
- final BytesRef thisPayload;
-
- if (readPositions) {
- final int code = prox.readVInt();
- position += code >>> 1;
-
- if ((code & 1) != 0) {
-
- // This position has a payload
- final int payloadLength = prox.readVInt();
-
- if (payload == null) {
- payload = new BytesRefBuilder();
- }
- payload.grow(payloadLength);
-
- prox.readBytes(payload.bytes(), 0, payloadLength);
- payload.setLength(payloadLength);
- thisPayload = payload.get();
-
- } else {
- thisPayload = null;
- }
-
- if (readOffsets) {
- final int startOffset = offset + prox.readVInt();
- final int endOffset = startOffset + prox.readVInt();
- if (writePositions) {
- if (writeOffsets) {
- assert startOffset >=0 && endOffset >= startOffset : "startOffset=" + startOffset + ",endOffset=" + endOffset + ",offset=" + offset;
- postingsConsumer.addPosition(position, thisPayload, startOffset, endOffset);
- } else {
- postingsConsumer.addPosition(position, thisPayload, -1, -1);
- }
- }
- offset = startOffset;
- } else if (writePositions) {
- postingsConsumer.addPosition(position, thisPayload, -1, -1);
- }
- }
- }
- }
- postingsConsumer.finishDoc();
- }
- termsConsumer.finishTerm(text, new TermStats(docFreq, writeTermFreq ? totalTermFreq : -1));
- sumTotalTermFreq += totalTermFreq;
- sumDocFreq += docFreq;
- }
-
- termsConsumer.finish(writeTermFreq ? sumTotalTermFreq : -1, sumDocFreq, visitedDocs.cardinality());
- }
}
Added: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java?rev=1620250&view=auto
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java (added)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/index/MappedMultiFields.java Mon Aug 25 01:52:08 2014
@@ -0,0 +1,146 @@
+package org.apache.lucene.index;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.util.Bits;
+
+import static org.apache.lucene.index.FilterAtomicReader.FilterFields;
+import static org.apache.lucene.index.FilterAtomicReader.FilterTerms;
+import static org.apache.lucene.index.FilterAtomicReader.FilterTermsEnum;
+
+/** A {@link Fields} implementation that merges multiple
+ * Fields into one, and maps around deleted documents.
+ * This is used for merging.
+ * @lucene.internal
+ */
+public class MappedMultiFields extends FilterFields {
+ final MergeState mergeState;
+
+ /** Create a new MappedMultiFields for merging, based on the supplied
+ * mergestate and merged view of terms. */
+ public MappedMultiFields(MergeState mergeState, MultiFields multiFields) {
+ super(multiFields);
+ this.mergeState = mergeState;
+ }
+
+ @Override
+ public Terms terms(String field) throws IOException {
+ MultiTerms terms = (MultiTerms) in.terms(field);
+ if (terms == null) {
+ return null;
+ } else {
+ return new MappedMultiTerms(mergeState, terms);
+ }
+ }
+
+ private static class MappedMultiTerms extends FilterTerms {
+ final MergeState mergeState;
+
+ public MappedMultiTerms(MergeState mergeState, MultiTerms multiTerms) {
+ super(multiTerms);
+ this.mergeState = mergeState;
+ }
+
+ @Override
+ public TermsEnum iterator(TermsEnum reuse) throws IOException {
+ // TODO: this is needed for codecs that aren't ghostbusters, can we improve this?
+ // should we have it in trunk too for safety?
+ TermsEnum iterator = in.iterator(reuse);
+ if (iterator instanceof MultiTermsEnum) {
+ return new MappedMultiTermsEnum(mergeState, (MultiTermsEnum) iterator);
+ } else {
+ return iterator;
+ }
+ }
+
+ @Override
+ public long size() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSumTotalTermFreq() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long getSumDocFreq() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getDocCount() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ private static class MappedMultiTermsEnum extends FilterTermsEnum {
+ final MergeState mergeState;
+
+ public MappedMultiTermsEnum(MergeState mergeState, MultiTermsEnum multiTermsEnum) {
+ super(multiTermsEnum);
+ this.mergeState = mergeState;
+ }
+
+ @Override
+ public int docFreq() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public long totalTermFreq() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ if (liveDocs != null) {
+ throw new IllegalArgumentException("liveDocs must be null");
+ }
+ MappingMultiDocsEnum mappingDocsEnum;
+ if (reuse instanceof MappingMultiDocsEnum) {
+ mappingDocsEnum = (MappingMultiDocsEnum) reuse;
+ } else {
+ mappingDocsEnum = new MappingMultiDocsEnum(mergeState);
+ }
+
+ MultiDocsEnum docsEnum = (MultiDocsEnum) in.docs(liveDocs, mappingDocsEnum.multiDocsEnum, flags);
+ mappingDocsEnum.reset(docsEnum);
+ return mappingDocsEnum;
+ }
+
+ @Override
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ if (liveDocs != null) {
+ throw new IllegalArgumentException("liveDocs must be null");
+ }
+ MappingMultiDocsAndPositionsEnum mappingDocsAndPositionsEnum;
+ if (reuse instanceof MappingMultiDocsAndPositionsEnum) {
+ mappingDocsAndPositionsEnum = (MappingMultiDocsAndPositionsEnum) reuse;
+ } else {
+ mappingDocsAndPositionsEnum = new MappingMultiDocsAndPositionsEnum(mergeState);
+ }
+
+ MultiDocsAndPositionsEnum docsAndPositionsEnum = (MultiDocsAndPositionsEnum) in.docsAndPositions(liveDocs, mappingDocsAndPositionsEnum.multiDocsAndPositionsEnum, flags);
+ mappingDocsAndPositionsEnum.reset(docsAndPositionsEnum);
+ return mappingDocsAndPositionsEnum;
+ }
+ }
+}
\ No newline at end of file