You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/10/05 16:43:51 UTC
svn commit: r1629501 [2/7] - in /lucene/dev/branches/branch_5x: ./
dev-tools/ lucene/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/
lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/
lucene/backward-codecs/src/java/...
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsFormat.java Sun Oct 5 14:43:47 2014
@@ -19,226 +19,25 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.PostingsWriterBase; // javadocs
import org.apache.lucene.codecs.blocktree.BlockTreeTermsReader;
-import org.apache.lucene.codecs.blocktree.BlockTreeTermsWriter;
-import org.apache.lucene.index.DocsEnum; // javadocs
-import org.apache.lucene.index.FieldInfo.IndexOptions; // javadocs
-import org.apache.lucene.index.FieldInfos; // javadocs
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.DataOutput; // javadocs
-import org.apache.lucene.util.fst.FST; // javadocs
/**
* Lucene 4.0 Postings format.
- * <p>
- * Files:
- * <ul>
- * <li><tt>.tim</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
- * <li><tt>.tip</tt>: <a href="#Termindex">Term Index</a></li>
- * <li><tt>.frq</tt>: <a href="#Frequencies">Frequencies</a></li>
- * <li><tt>.prx</tt>: <a href="#Positions">Positions</a></li>
- * </ul>
- * </p>
- * <p>
- * <a name="Termdictionary" id="Termdictionary"></a>
- * <h3>Term Dictionary</h3>
- *
- * <p>The .tim file contains the list of terms in each
- * field along with per-term statistics (such as docfreq)
- * and pointers to the frequencies, positions and
- * skip data in the .frq and .prx files.
- * See {@link BlockTreeTermsWriter} for more details on the format.
- * </p>
- *
- * <p>NOTE: The term dictionary can plug into different postings implementations:
- * the postings writer/reader are actually responsible for encoding
- * and decoding the Postings Metadata and Term Metadata sections described here:</p>
- * <ul>
- * <li>Postings Metadata --> Header, SkipInterval, MaxSkipLevels, SkipMinimum</li>
- * <li>Term Metadata --> FreqDelta, SkipDelta?, ProxDelta?
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>SkipInterval,MaxSkipLevels,SkipMinimum --> {@link DataOutput#writeInt Uint32}</li>
- * <li>SkipDelta,FreqDelta,ProxDelta --> {@link DataOutput#writeVLong VLong}</li>
- * </ul>
- * <p>Notes:</p>
- * <ul>
- * <li>Header is a {@link CodecUtil#writeHeader CodecHeader} storing the version information
- * for the postings.</li>
- * <li>SkipInterval is the fraction of TermDocs stored in skip tables. It is used to accelerate
- * {@link DocsEnum#advance(int)}. Larger values result in smaller indexes, greater
- * acceleration, but fewer accelerable cases, while smaller values result in bigger indexes,
- * less acceleration (in case of a small value for MaxSkipLevels) and more accelerable cases.
- * </li>
- * <li>MaxSkipLevels is the max. number of skip levels stored for each term in the .frq file. A
- * low value results in smaller indexes but less acceleration, a larger value results in
- * slightly larger indexes but greater acceleration. See format of .frq file for more
- * information about skip levels.</li>
- * <li>SkipMinimum is the minimum document frequency a term must have in order to write any
- * skip data at all.</li>
- * <li>FreqDelta determines the position of this term's TermFreqs within the .frq
- * file. In particular, it is the difference between the position of this term's
- * data in that file and the position of the previous term's data (or zero, for
- * the first term in the block).</li>
- * <li>ProxDelta determines the position of this term's TermPositions within the
- * .prx file. In particular, it is the difference between the position of this
- * term's data in that file and the position of the previous term's data (or zero,
- * for the first term in the block. For fields that omit position data, this will
- * be 0 since prox information is not stored.</li>
- * <li>SkipDelta determines the position of this term's SkipData within the .frq
- * file. In particular, it is the number of bytes after TermFreqs that the
- * SkipData starts. In other words, it is the length of the TermFreq data.
- * SkipDelta is only stored if DocFreq is not smaller than SkipMinimum.</li>
- * </ul>
- * <a name="Termindex" id="Termindex"></a>
- * <h3>Term Index</h3>
- * <p>The .tip file contains an index into the term dictionary, so that it can be
- * accessed randomly. See {@link BlockTreeTermsWriter} for more details on the format.</p>
- * <a name="Frequencies" id="Frequencies"></a>
- * <h3>Frequencies</h3>
- * <p>The .frq file contains the lists of documents which contain each term, along
- * with the frequency of the term in that document (except when frequencies are
- * omitted: {@link IndexOptions#DOCS_ONLY}).</p>
- * <ul>
- * <li>FreqFile (.frq) --> Header, <TermFreqs, SkipData?> <sup>TermCount</sup></li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>TermFreqs --> <TermFreq> <sup>DocFreq</sup></li>
- * <li>TermFreq --> DocDelta[, Freq?]</li>
- * <li>SkipData --> <<SkipLevelLength, SkipLevel>
- * <sup>NumSkipLevels-1</sup>, SkipLevel> <SkipDatum></li>
- * <li>SkipLevel --> <SkipDatum> <sup>DocFreq/(SkipInterval^(Level +
- * 1))</sup></li>
- * <li>SkipDatum -->
- * DocSkip,PayloadLength?,OffsetLength?,FreqSkip,ProxSkip,SkipChildLevelPointer?</li>
- * <li>DocDelta,Freq,DocSkip,PayloadLength,OffsetLength,FreqSkip,ProxSkip --> {@link DataOutput#writeVInt VInt}</li>
- * <li>SkipChildLevelPointer --> {@link DataOutput#writeVLong VLong}</li>
- * </ul>
- * <p>TermFreqs are ordered by term (the term is implicit, from the term dictionary).</p>
- * <p>TermFreq entries are ordered by increasing document number.</p>
- * <p>DocDelta: if frequencies are indexed, this determines both the document
- * number and the frequency. In particular, DocDelta/2 is the difference between
- * this document number and the previous document number (or zero when this is the
- * first document in a TermFreqs). When DocDelta is odd, the frequency is one.
- * When DocDelta is even, the frequency is read as another VInt. If frequencies
- * are omitted, DocDelta contains the gap (not multiplied by 2) between document
- * numbers and no frequency information is stored.</p>
- * <p>For example, the TermFreqs for a term which occurs once in document seven
- * and three times in document eleven, with frequencies indexed, would be the
- * following sequence of VInts:</p>
- * <p>15, 8, 3</p>
- * <p>If frequencies were omitted ({@link IndexOptions#DOCS_ONLY}) it would be this
- * sequence of VInts instead:</p>
- * <p>7,4</p>
- * <p>DocSkip records the document number before every SkipInterval <sup>th</sup>
- * document in TermFreqs. If payloads and offsets are disabled for the term's field, then
- * DocSkip represents the difference from the previous value in the sequence. If
- * payloads and/or offsets are enabled for the term's field, then DocSkip/2 represents the
- * difference from the previous value in the sequence. In this case when
- * DocSkip is odd, then PayloadLength and/or OffsetLength are stored indicating the length of
- * the last payload/offset before the SkipInterval<sup>th</sup> document in TermPositions.</p>
- * <p>PayloadLength indicates the length of the last payload.</p>
- * <p>OffsetLength indicates the length of the last offset (endOffset-startOffset).</p>
- * <p>
- * FreqSkip and ProxSkip record the position of every SkipInterval <sup>th</sup>
- * entry in FreqFile and ProxFile, respectively. File positions are relative to
- * the start of TermFreqs and Positions, to the previous SkipDatum in the
- * sequence.</p>
- * <p>For example, if DocFreq=35 and SkipInterval=16, then there are two SkipData
- * entries, containing the 15 <sup>th</sup> and 31 <sup>st</sup> document numbers
- * in TermFreqs. The first FreqSkip names the number of bytes after the beginning
- * of TermFreqs that the 16 <sup>th</sup> SkipDatum starts, and the second the
- * number of bytes after that that the 32 <sup>nd</sup> starts. The first ProxSkip
- * names the number of bytes after the beginning of Positions that the 16
- * <sup>th</sup> SkipDatum starts, and the second the number of bytes after that
- * that the 32 <sup>nd</sup> starts.</p>
- * <p>Each term can have multiple skip levels. The amount of skip levels for a
- * term is NumSkipLevels = Min(MaxSkipLevels,
- * floor(log(DocFreq/log(SkipInterval)))). The number of SkipData entries for a
- * skip level is DocFreq/(SkipInterval^(Level + 1)), whereas the lowest skip level
- * is Level=0.<br>
- * Example: SkipInterval = 4, MaxSkipLevels = 2, DocFreq = 35. Then skip level 0
- * has 8 SkipData entries, containing the 3<sup>rd</sup>, 7<sup>th</sup>,
- * 11<sup>th</sup>, 15<sup>th</sup>, 19<sup>th</sup>, 23<sup>rd</sup>,
- * 27<sup>th</sup>, and 31<sup>st</sup> document numbers in TermFreqs. Skip level
- * 1 has 2 SkipData entries, containing the 15<sup>th</sup> and 31<sup>st</sup>
- * document numbers in TermFreqs.<br>
- * The SkipData entries on all upper levels > 0 contain a SkipChildLevelPointer
- * referencing the corresponding SkipData entry in level-1. In the example has
- * entry 15 on level 1 a pointer to entry 15 on level 0 and entry 31 on level 1 a
- * pointer to entry 31 on level 0.
- * </p>
- * <a name="Positions" id="Positions"></a>
- * <h3>Positions</h3>
- * <p>The .prx file contains the lists of positions that each term occurs at
- * within documents. Note that fields omitting positional data do not store
- * anything into this file, and if all fields in the index omit positional data
- * then the .prx file will not exist.</p>
- * <ul>
- * <li>ProxFile (.prx) --> Header, <TermPositions> <sup>TermCount</sup></li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>TermPositions --> <Positions> <sup>DocFreq</sup></li>
- * <li>Positions --> <PositionDelta,PayloadLength?,OffsetDelta?,OffsetLength?,PayloadData?> <sup>Freq</sup></li>
- * <li>PositionDelta,OffsetDelta,OffsetLength,PayloadLength --> {@link DataOutput#writeVInt VInt}</li>
- * <li>PayloadData --> {@link DataOutput#writeByte byte}<sup>PayloadLength</sup></li>
- * </ul>
- * <p>TermPositions are ordered by term (the term is implicit, from the term dictionary).</p>
- * <p>Positions entries are ordered by increasing document number (the document
- * number is implicit from the .frq file).</p>
- * <p>PositionDelta is, if payloads are disabled for the term's field, the
- * difference between the position of the current occurrence in the document and
- * the previous occurrence (or zero, if this is the first occurrence in this
- * document). If payloads are enabled for the term's field, then PositionDelta/2
- * is the difference between the current and the previous position. If payloads
- * are enabled and PositionDelta is odd, then PayloadLength is stored, indicating
- * the length of the payload at the current term position.</p>
- * <p>For example, the TermPositions for a term which occurs as the fourth term in
- * one document, and as the fifth and ninth term in a subsequent document, would
- * be the following sequence of VInts (payloads disabled):</p>
- * <p>4, 5, 4</p>
- * <p>PayloadData is metadata associated with the current term position. If
- * PayloadLength is stored at the current position, then it indicates the length
- * of this payload. If PayloadLength is not stored, then this payload has the same
- * length as the payload at the previous position.</p>
- * <p>OffsetDelta/2 is the difference between this position's startOffset from the
- * previous occurrence (or zero, if this is the first occurrence in this document).
- * If OffsetDelta is odd, then the length (endOffset-startOffset) differs from the
- * previous occurrence and an OffsetLength follows. Offset data is only written for
- * {@link IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS}.</p>
- *
- * @deprecated Only for reading old 4.0 segments */
-
-// TODO: this class could be created by wrapping
-// BlockTreeTermsDict around Lucene40PostingsBaseFormat; ie
-// we should not duplicate the code from that class here:
+ * @deprecated Only for reading old 4.0 segments
+ */
@Deprecated
public class Lucene40PostingsFormat extends PostingsFormat {
- /** minimum items (terms or sub-blocks) per block for BlockTree */
- protected final int minBlockSize;
- /** maximum items (terms or sub-blocks) per block for BlockTree */
- protected final int maxBlockSize;
-
/** Creates {@code Lucene40PostingsFormat} with default
* settings. */
public Lucene40PostingsFormat() {
- this(BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
- }
-
- /** Creates {@code Lucene40PostingsFormat} with custom
- * values for {@code minBlockSize} and {@code
- * maxBlockSize} passed to block terms dictionary.
- * @see BlockTreeTermsWriter#BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int) */
- private Lucene40PostingsFormat(int minBlockSize, int maxBlockSize) {
super("Lucene40");
- this.minBlockSize = minBlockSize;
- assert minBlockSize > 1;
- this.maxBlockSize = maxBlockSize;
}
@Override
@@ -247,7 +46,7 @@ public class Lucene40PostingsFormat exte
}
@Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ public final FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
PostingsReaderBase postings = new Lucene40PostingsReader(state.directory, state.fieldInfos, state.segmentInfo, state.context, state.segmentSuffix);
boolean success = false;
@@ -276,6 +75,6 @@ public class Lucene40PostingsFormat exte
@Override
public String toString() {
- return getName() + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")";
+ return getName();
}
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java Sun Oct 5 14:43:47 2014
@@ -43,13 +43,10 @@ import org.apache.lucene.util.BytesRefBu
import org.apache.lucene.util.IOUtils;
/**
- * Concrete class that reads the 4.0 frq/prox
- * postings format.
- *
- * @see Lucene40PostingsFormat
- * @deprecated Only for reading old 4.0 segments */
+ * Reader for 4.0 postings format
+ * @deprecated Only for reading old 4.0 segments */
@Deprecated
-public class Lucene40PostingsReader extends PostingsReaderBase {
+final class Lucene40PostingsReader extends PostingsReaderBase {
final static String TERMS_CODEC = "Lucene40PostingsWriterTerms";
final static String FRQ_CODEC = "Lucene40PostingsWriterFrq";
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoFormat.java Sun Oct 5 14:43:47 2014
@@ -17,57 +17,14 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.SegmentInfoReader;
import org.apache.lucene.codecs.SegmentInfoWriter;
-import org.apache.lucene.index.IndexWriter; // javadocs
-import org.apache.lucene.index.SegmentInfo; // javadocs
-import org.apache.lucene.index.SegmentInfos; // javadocs
-import org.apache.lucene.store.DataOutput; // javadocs
+import org.apache.lucene.index.SegmentInfo;
/**
* Lucene 4.0 Segment info format.
- * <p>
- * Files:
- * <ul>
- * <li><tt>.si</tt>: Header, SegVersion, SegSize, IsCompoundFile, Diagnostics, Attributes, Files
- * </ul>
- * </p>
- * Data types:
- * <p>
- * <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>SegSize --> {@link DataOutput#writeInt Int32}</li>
- * <li>SegVersion --> {@link DataOutput#writeString String}</li>
- * <li>Files --> {@link DataOutput#writeStringSet Set<String>}</li>
- * <li>Diagnostics, Attributes --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
- * <li>IsCompoundFile --> {@link DataOutput#writeByte Int8}</li>
- * </ul>
- * </p>
- * Field Descriptions:
- * <p>
- * <ul>
- * <li>SegVersion is the code version that created the segment.</li>
- * <li>SegSize is the number of documents contained in the segment index.</li>
- * <li>IsCompoundFile records whether the segment is written as a compound file or
- * not. If this is -1, the segment is not a compound file. If it is 1, the segment
- * is a compound file.</li>
- * <li>Checksum contains the CRC32 checksum of all bytes in the segments_N file up
- * until the checksum. This is used to verify integrity of the file on opening the
- * index.</li>
- * <li>The Diagnostics Map is privately written by {@link IndexWriter}, as a debugging aid,
- * for each segment it creates. It includes metadata like the current Lucene
- * version, OS, Java version, why the segment was created (merge, flush,
- * addIndexes), etc.</li>
- * <li>Attributes: a key-value map of codec-private attributes.</li>
- * <li>Files is a list of files referred to by this segment.</li>
- * </ul>
- * </p>
- *
- * @see SegmentInfos
- * @lucene.experimental
- * @deprecated Only for reading old 4.0-4.5 segments, and supporting IndexWriter.addIndexes
+ * @deprecated Only for reading old 4.0-4.5 segments
*/
@Deprecated
public class Lucene40SegmentInfoFormat extends SegmentInfoFormat {
@@ -78,7 +35,7 @@ public class Lucene40SegmentInfoFormat e
}
@Override
- public SegmentInfoReader getSegmentInfoReader() {
+ public final SegmentInfoReader getSegmentInfoReader() {
return reader;
}
@@ -88,7 +45,7 @@ public class Lucene40SegmentInfoFormat e
}
/** File extension used to store {@link SegmentInfo}. */
- public final static String SI_EXTENSION = "si";
+ static final String SI_EXTENSION = "si";
static final String CODEC_NAME = "Lucene40SegmentInfo";
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SegmentInfoReader.java Sun Oct 5 14:43:47 2014
@@ -34,14 +34,11 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.Version;
/**
- * Lucene 4.0 implementation of {@link SegmentInfoReader}.
- *
- * @see Lucene40SegmentInfoFormat
- * @lucene.experimental
+ * Lucene 4.0 SI reader
* @deprecated Only for reading old 4.0-4.5 segments
*/
@Deprecated
-public class Lucene40SegmentInfoReader extends SegmentInfoReader {
+final class Lucene40SegmentInfoReader extends SegmentInfoReader {
/** Sole constructor. */
public Lucene40SegmentInfoReader() {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40SkipListReader.java Sun Oct 5 14:43:47 2014
@@ -24,14 +24,11 @@ import org.apache.lucene.codecs.MultiLev
import org.apache.lucene.store.IndexInput;
/**
- * Implements the skip list reader for the 4.0 posting list format
- * that stores positions and payloads.
- *
- * @see Lucene40PostingsFormat
+ * Lucene 4.0 skiplist reader
* @deprecated Only for reading old 4.0 segments
*/
@Deprecated
-public class Lucene40SkipListReader extends MultiLevelSkipListReader {
+final class Lucene40SkipListReader extends MultiLevelSkipListReader {
private boolean currentFieldStoresPayloads;
private boolean currentFieldStoresOffsets;
private long freqPointer[];
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsFormat.java Sun Oct 5 14:43:47 2014
@@ -19,66 +19,18 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.DataOutput; // javadocs
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
/**
* Lucene 4.0 Stored Fields Format.
- * <p>Stored fields are represented by two files:</p>
- * <ol>
- * <li><a name="field_index" id="field_index"></a>
- * <p>The field index, or <tt>.fdx</tt> file.</p>
- * <p>This is used to find the location within the field data file of the fields
- * of a particular document. Because it contains fixed-length data, this file may
- * be easily randomly accessed. The position of document <i>n</i> 's field data is
- * the {@link DataOutput#writeLong Uint64} at <i>n*8</i> in this file.</p>
- * <p>This contains, for each document, a pointer to its field data, as
- * follows:</p>
- * <ul>
- * <li>FieldIndex (.fdx) --> <Header>, <FieldValuesPosition> <sup>SegSize</sup></li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>FieldValuesPosition --> {@link DataOutput#writeLong Uint64}</li>
- * </ul>
- * </li>
- * <li>
- * <p><a name="field_data" id="field_data"></a>The field data, or <tt>.fdt</tt> file.</p>
- * <p>This contains the stored fields of each document, as follows:</p>
- * <ul>
- * <li>FieldData (.fdt) --> <Header>, <DocFieldData> <sup>SegSize</sup></li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>DocFieldData --> FieldCount, <FieldNum, Bits, Value>
- * <sup>FieldCount</sup></li>
- * <li>FieldCount --> {@link DataOutput#writeVInt VInt}</li>
- * <li>FieldNum --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Bits --> {@link DataOutput#writeByte Byte}</li>
- * <ul>
- * <li>low order bit reserved.</li>
- * <li>second bit is one for fields containing binary data</li>
- * <li>third bit reserved.</li>
- * <li>4th to 6th bit (mask: 0x7<<3) define the type of a numeric field:
- * <ul>
- * <li>all bits in mask are cleared if no numeric field at all</li>
- * <li>1<<3: Value is Int</li>
- * <li>2<<3: Value is Long</li>
- * <li>3<<3: Value is Int as Float (as of {@link Float#intBitsToFloat(int)}</li>
- * <li>4<<3: Value is Long as Double (as of {@link Double#longBitsToDouble(long)}</li>
- * </ul>
- * </li>
- * </ul>
- * <li>Value --> String | BinaryValue | Int | Long (depending on Bits)</li>
- * <li>BinaryValue --> ValueSize, <{@link DataOutput#writeByte Byte}>^ValueSize</li>
- * <li>ValueSize --> {@link DataOutput#writeVInt VInt}</li>
- * </li>
- * </ul>
- * </ol>
- * @lucene.experimental */
+ * @deprecated only for reading 4.0 segments */
+@Deprecated
public class Lucene40StoredFieldsFormat extends StoredFieldsFormat {
/** Sole constructor. */
@@ -86,7 +38,7 @@ public class Lucene40StoredFieldsFormat
}
@Override
- public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
+ public final StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
FieldInfos fn, IOContext context) throws IOException {
return new Lucene40StoredFieldsReader(directory, si, fn, context);
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsReader.java Sun Oct 5 14:43:47 2014
@@ -40,14 +40,11 @@ import java.nio.charset.StandardCharsets
import java.util.Collections;
/**
- * Class responsible for access to stored document fields.
- * <p/>
- * It uses <segment>.fdt and <segment>.fdx; files.
- *
- * @see Lucene40StoredFieldsFormat
- * @lucene.internal
+ * Reader for 4.0 stored fields
+ * @deprecated only for reading 4.0 segments
*/
-public final class Lucene40StoredFieldsReader extends StoredFieldsReader implements Cloneable, Closeable {
+@Deprecated
+final class Lucene40StoredFieldsReader extends StoredFieldsReader implements Cloneable, Closeable {
// NOTE: bit 0 is free here! You can steal it!
static final int FIELD_IS_BINARY = 1 << 1;
@@ -76,10 +73,10 @@ public final class Lucene40StoredFieldsR
/** Extension of stored fields file */
- public static final String FIELDS_EXTENSION = "fdt";
+ static final String FIELDS_EXTENSION = "fdt";
/** Extension of stored fields index file */
- public static final String FIELDS_INDEX_EXTENSION = "fdx";
+ static final String FIELDS_INDEX_EXTENSION = "fdx";
private static final long RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(Lucene40StoredFieldsReader.class);
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsFormat.java Sun Oct 5 14:43:47 2014
@@ -19,100 +19,19 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.TermVectorsFormat;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.TermVectorsWriter;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.DataOutput; // javadocs
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
/**
* Lucene 4.0 Term Vectors format.
- * <p>Term Vector support is an optional on a field by field basis. It consists of
- * 3 files.</p>
- * <ol>
- * <li><a name="tvx" id="tvx"></a>
- * <p>The Document Index or .tvx file.</p>
- * <p>For each document, this stores the offset into the document data (.tvd) and
- * field data (.tvf) files.</p>
- * <p>DocumentIndex (.tvx) --> Header,<DocumentPosition,FieldPosition>
- * <sup>NumDocs</sup></p>
- * <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>DocumentPosition --> {@link DataOutput#writeLong UInt64} (offset in the .tvd file)</li>
- * <li>FieldPosition --> {@link DataOutput#writeLong UInt64} (offset in the .tvf file)</li>
- * </ul>
- * </li>
- * <li><a name="tvd" id="tvd"></a>
- * <p>The Document or .tvd file.</p>
- * <p>This contains, for each document, the number of fields, a list of the fields
- * with term vector info and finally a list of pointers to the field information
- * in the .tvf (Term Vector Fields) file.</p>
- * <p>The .tvd file is used to map out the fields that have term vectors stored
- * and where the field information is in the .tvf file.</p>
- * <p>Document (.tvd) --> Header,<NumFields, FieldNums,
- * FieldPositions> <sup>NumDocs</sup></p>
- * <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>NumFields --> {@link DataOutput#writeVInt VInt}</li>
- * <li>FieldNums --> <FieldNumDelta> <sup>NumFields</sup></li>
- * <li>FieldNumDelta --> {@link DataOutput#writeVInt VInt}</li>
- * <li>FieldPositions --> <FieldPositionDelta> <sup>NumFields-1</sup></li>
- * <li>FieldPositionDelta --> {@link DataOutput#writeVLong VLong}</li>
- * </ul>
- * </li>
- * <li><a name="tvf" id="tvf"></a>
- * <p>The Field or .tvf file.</p>
- * <p>This file contains, for each field that has a term vector stored, a list of
- * the terms, their frequencies and, optionally, position, offset, and payload
- * information.</p>
- * <p>Field (.tvf) --> Header,<NumTerms, Flags, TermFreqs>
- * <sup>NumFields</sup></p>
- * <ul>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>NumTerms --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Flags --> {@link DataOutput#writeByte Byte}</li>
- * <li>TermFreqs --> <TermText, TermFreq, Positions?, PayloadData?, Offsets?>
- * <sup>NumTerms</sup></li>
- * <li>TermText --> <PrefixLength, Suffix></li>
- * <li>PrefixLength --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Suffix --> {@link DataOutput#writeString String}</li>
- * <li>TermFreq --> {@link DataOutput#writeVInt VInt}</li>
- * <li>Positions --> <PositionDelta PayloadLength?><sup>TermFreq</sup></li>
- * <li>PositionDelta --> {@link DataOutput#writeVInt VInt}</li>
- * <li>PayloadLength --> {@link DataOutput#writeVInt VInt}</li>
- * <li>PayloadData --> {@link DataOutput#writeByte Byte}<sup>NumPayloadBytes</sup></li>
- * <li>Offsets --> <{@link DataOutput#writeVInt VInt}, {@link DataOutput#writeVInt VInt}><sup>TermFreq</sup></li>
- * </ul>
- * <p>Notes:</p>
- * <ul>
- * <li>Flags byte stores whether this term vector has position, offset, payload.
- * information stored.</li>
- * <li>Term byte prefixes are shared. The PrefixLength is the number of initial
- * bytes from the previous term which must be pre-pended to a term's suffix
- * in order to form the term's bytes. Thus, if the previous term's text was "bone"
- * and the term is "boy", the PrefixLength is two and the suffix is "y".</li>
- * <li>PositionDelta is, if payloads are disabled for the term's field, the
- * difference between the position of the current occurrence in the document and
- * the previous occurrence (or zero, if this is the first occurrence in this
- * document). If payloads are enabled for the term's field, then PositionDelta/2
- * is the difference between the current and the previous position. If payloads
- * are enabled and PositionDelta is odd, then PayloadLength is stored, indicating
- * the length of the payload at the current term position.</li>
- * <li>PayloadData is metadata associated with a term position. If
- * PayloadLength is stored at the current position, then it indicates the length
- * of this payload. If PayloadLength is not stored, then this payload has the same
- * length as the payload at the previous position. PayloadData encodes the
- * concatenated bytes for all of a terms occurrences.</li>
- * <li>Offsets are stored as delta encoded VInts. The first VInt is the
- * startOffset, the second is the endOffset.</li>
- * </ul>
- * </li>
- * </ol>
+ * @deprecated only for reading 4.0 and 4.1 segments
*/
+@Deprecated
public class Lucene40TermVectorsFormat extends TermVectorsFormat {
/** Sole constructor. */
@@ -120,7 +39,7 @@ public class Lucene40TermVectorsFormat e
}
@Override
- public TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException {
+ public final TermVectorsReader vectorsReader(Directory directory, SegmentInfo segmentInfo, FieldInfos fieldInfos, IOContext context) throws IOException {
return new Lucene40TermVectorsReader(directory, segmentInfo, fieldInfos, context);
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java Sun Oct 5 14:43:47 2014
@@ -48,12 +48,10 @@ import org.apache.lucene.util.IOUtils;
/**
* Lucene 4.0 Term Vectors reader.
- * <p>
- * It reads .tvd, .tvf, and .tvx files.
- *
- * @see Lucene40TermVectorsFormat
+ * @deprecated only for reading 4.0 and 4.1 segments
*/
-public class Lucene40TermVectorsReader extends TermVectorsReader implements Closeable {
+@Deprecated
+final class Lucene40TermVectorsReader extends TermVectorsReader implements Closeable {
static final byte STORE_POSITIONS_WITH_TERMVECTOR = 0x1;
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene41/Lucene41Codec.java Sun Oct 5 14:43:47 2014
@@ -17,21 +17,17 @@ package org.apache.lucene.codecs.lucene4
* limitations under the License.
*/
-import java.io.IOException;
-
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
-import org.apache.lucene.codecs.StoredFieldsWriter;
import org.apache.lucene.codecs.TermVectorsFormat;
-import org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat;
-import org.apache.lucene.codecs.compressing.CompressionMode;
+import org.apache.lucene.codecs.lucene40.Lucene40CompoundFormat;
import org.apache.lucene.codecs.lucene40.Lucene40DocValuesFormat;
import org.apache.lucene.codecs.lucene40.Lucene40FieldInfosFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
@@ -39,33 +35,19 @@ import org.apache.lucene.codecs.lucene40
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
/**
- * Implements the Lucene 4.1 index format, with configurable per-field postings formats.
- * <p>
- * If you want to reuse functionality of this codec in another codec, extend
- * {@link FilterCodec}.
- *
- * @see org.apache.lucene.codecs.lucene41 package documentation for file format details.
- * @deprecated Only for reading old 4.0 segments
- * @lucene.experimental
+ * Implements the Lucene 4.1 index format
+ * @deprecated Only for reading old 4.1 segments
*/
@Deprecated
public class Lucene41Codec extends Codec {
- // TODO: slightly evil
- private final StoredFieldsFormat fieldsFormat = new CompressingStoredFieldsFormat("Lucene41StoredFields", CompressionMode.FAST, 1 << 14) {
- @Override
- public StoredFieldsWriter fieldsWriter(Directory directory, SegmentInfo si, IOContext context) throws IOException {
- throw new UnsupportedOperationException("this codec can only be used for reading");
- }
- };
+ private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
private final TermVectorsFormat vectorsFormat = new Lucene40TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene40FieldInfosFormat();
private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene40CompoundFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
@@ -79,7 +61,6 @@ public class Lucene41Codec extends Codec
super("Lucene41");
}
- // TODO: slightly evil
@Override
public StoredFieldsFormat storedFieldsFormat() {
return fieldsFormat;
@@ -109,6 +90,11 @@ public class Lucene41Codec extends Codec
public final LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}
+
+ @Override
+ public CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene410/Lucene410Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene410/Lucene410Codec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene410/Lucene410Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene410/Lucene410Codec.java Sun Oct 5 14:43:47 2014
@@ -20,9 +20,9 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40CompoundFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
import org.apache.lucene.codecs.lucene42.Lucene42TermVectorsFormat;
@@ -41,24 +42,17 @@ import org.apache.lucene.codecs.perfield
import org.apache.lucene.index.SegmentWriteState;
/**
- * Implements the Lucene 4.10 index format, with configurable per-field postings
- * and docvalues formats.
- * <p>
- * If you want to reuse functionality of this codec in another codec, extend
- * {@link FilterCodec}.
- *
- * @see org.apache.lucene.codecs.lucene410 package documentation for file format details.
- * @lucene.experimental
+ * Implements the Lucene 4.10 codec
+ * @deprecated only for reading old 4.10 segments
*/
-// NOTE: if we make largish changes in a minor release, easier to just make Lucene411Codec or whatever
-// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
-// (it writes a minor version, etc).
+@Deprecated
public class Lucene410Codec extends Codec {
private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
private final TermVectorsFormat vectorsFormat = new Lucene42TermVectorsFormat();
private final FieldInfosFormat fieldInfosFormat = new Lucene46FieldInfosFormat();
private final SegmentInfoFormat segmentInfosFormat = new Lucene46SegmentInfoFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene40CompoundFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
@@ -108,6 +102,11 @@ public class Lucene410Codec extends Code
public final LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}
+
+ @Override
+ public CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42Codec.java Sun Oct 5 14:43:47 2014
@@ -20,9 +20,9 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40CompoundFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
@@ -38,19 +39,9 @@ import org.apache.lucene.codecs.perfield
import org.apache.lucene.index.SegmentWriteState;
/**
- * Implements the Lucene 4.2 index format, with configurable per-field postings
- * and docvalues formats.
- * <p>
- * If you want to reuse functionality of this codec in another codec, extend
- * {@link FilterCodec}.
- *
- * @see org.apache.lucene.codecs.lucene42 package documentation for file format details.
- * @lucene.experimental
+ * Implements the Lucene 4.2 index format
* @deprecated Only for reading old 4.2 segments
*/
-// NOTE: if we make largish changes in a minor release, easier to just make Lucene43Codec or whatever
-// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
-// (it writes a minor version, etc).
@Deprecated
public class Lucene42Codec extends Codec {
private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
@@ -58,6 +49,7 @@ public class Lucene42Codec extends Codec
private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat();
private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene40CompoundFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
@@ -80,12 +72,12 @@ public class Lucene42Codec extends Codec
}
@Override
- public final StoredFieldsFormat storedFieldsFormat() {
+ public StoredFieldsFormat storedFieldsFormat() {
return fieldsFormat;
}
@Override
- public final TermVectorsFormat termVectorsFormat() {
+ public TermVectorsFormat termVectorsFormat() {
return vectorsFormat;
}
@@ -108,6 +100,11 @@ public class Lucene42Codec extends Codec
public final LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}
+
+ @Override
+ public CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesFormat.java Sun Oct 5 14:43:47 2014
@@ -19,119 +19,22 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
-import org.apache.lucene.util.packed.BlockPackedWriter;
/**
* Lucene 4.2 DocValues format.
- * <p>
- * Encodes the four per-document value types (Numeric,Binary,Sorted,SortedSet) with seven basic strategies.
- * <p>
- * <ul>
- * <li>Delta-compressed Numerics: per-document integers written in blocks of 4096. For each block
- * the minimum value is encoded, and each entry is a delta from that minimum value.
- * <li>Table-compressed Numerics: when the number of unique values is very small, a lookup table
- * is written instead. Each per-document entry is instead the ordinal to this table.
- * <li>Uncompressed Numerics: when all values would fit into a single byte, and the
- * <code>acceptableOverheadRatio</code> would pack values into 8 bits per value anyway, they
- * are written as absolute values (with no indirection or packing) for performance.
- * <li>GCD-compressed Numerics: when all numbers share a common divisor, such as dates, the greatest
- * common denominator (GCD) is computed, and quotients are stored using Delta-compressed Numerics.
- * <li>Fixed-width Binary: one large concatenated byte[] is written, along with the fixed length.
- * Each document's value can be addressed by maxDoc*length.
- * <li>Variable-width Binary: one large concatenated byte[] is written, along with end addresses
- * for each document. The addresses are written in blocks of 4096, with the current absolute
- * start for the block, and the average (expected) delta per entry. For each document the
- * deviation from the delta (actual - expected) is written.
- * <li>Sorted: an FST mapping deduplicated terms to ordinals is written, along with the per-document
- * ordinals written using one of the numeric strategies above.
- * <li>SortedSet: an FST mapping deduplicated terms to ordinals is written, along with the per-document
- * ordinal list written using one of the binary strategies above.
- * </ul>
- * <p>
- * Files:
- * <ol>
- * <li><tt>.dvd</tt>: DocValues data</li>
- * <li><tt>.dvm</tt>: DocValues metadata</li>
- * </ol>
- * <ol>
- * <li><a name="dvm" id="dvm"></a>
- * <p>The DocValues metadata or .dvm file.</p>
- * <p>For DocValues field, this stores metadata, such as the offset into the
- * DocValues data (.dvd)</p>
- * <p>DocValues metadata (.dvm) --> Header,<FieldNumber,EntryType,Entry><sup>NumFields</sup>,Footer</p>
- * <ul>
- * <li>Entry --> NumericEntry | BinaryEntry | SortedEntry</li>
- * <li>NumericEntry --> DataOffset,CompressionType,PackedVersion</li>
- * <li>BinaryEntry --> DataOffset,DataLength,MinLength,MaxLength,PackedVersion?,BlockSize?</li>
- * <li>SortedEntry --> DataOffset,ValueCount</li>
- * <li>FieldNumber,PackedVersion,MinLength,MaxLength,BlockSize,ValueCount --> {@link DataOutput#writeVInt VInt}</li>
- * <li>DataOffset,DataLength --> {@link DataOutput#writeLong Int64}</li>
- * <li>EntryType,CompressionType --> {@link DataOutput#writeByte Byte}</li>
- * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}</li>
- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
- * </ul>
- * <p>Sorted fields have two entries: a SortedEntry with the FST metadata,
- * and an ordinary NumericEntry for the document-to-ord metadata.</p>
- * <p>SortedSet fields have two entries: a SortedEntry with the FST metadata,
- * and an ordinary BinaryEntry for the document-to-ord-list metadata.</p>
- * <p>FieldNumber of -1 indicates the end of metadata.</p>
- * <p>EntryType is a 0 (NumericEntry), 1 (BinaryEntry, or 2 (SortedEntry)</p>
- * <p>DataOffset is the pointer to the start of the data in the DocValues data (.dvd)</p>
- * <p>CompressionType indicates how Numeric values will be compressed:
- * <ul>
- * <li>0 --> delta-compressed. For each block of 4096 integers, every integer is delta-encoded
- * from the minimum value within the block.
- * <li>1 --> table-compressed. When the number of unique numeric values is small and it would save space,
- * a lookup table of unique values is written, followed by the ordinal for each document.
- * <li>2 --> uncompressed. When the <code>acceptableOverheadRatio</code> parameter would upgrade the number
- * of bits required to 8, and all values fit in a byte, these are written as absolute binary values
- * for performance.
- * <li>3 -->, gcd-compressed. When all integers share a common divisor, only quotients are stored
- * using blocks of delta-encoded ints.
- * </ul>
- * <p>MinLength and MaxLength represent the min and max byte[] value lengths for Binary values.
- * If they are equal, then all values are of a fixed size, and can be addressed as DataOffset + (docID * length).
- * Otherwise, the binary values are of variable size, and packed integer metadata (PackedVersion,BlockSize)
- * is written for the addresses.
- * <li><a name="dvd" id="dvd"></a>
- * <p>The DocValues data or .dvd file.</p>
- * <p>For DocValues field, this stores the actual per-document data (the heavy-lifting)</p>
- * <p>DocValues data (.dvd) --> Header,<NumericData | BinaryData | SortedData><sup>NumFields</sup>,Footer</p>
- * <ul>
- * <li>NumericData --> DeltaCompressedNumerics | TableCompressedNumerics | UncompressedNumerics | GCDCompressedNumerics</li>
- * <li>BinaryData --> {@link DataOutput#writeByte Byte}<sup>DataLength</sup>,Addresses</li>
- * <li>SortedData --> {@link FST FST<Int64>}</li>
- * <li>DeltaCompressedNumerics --> {@link BlockPackedWriter BlockPackedInts(blockSize=4096)}</li>
- * <li>TableCompressedNumerics --> TableSize,{@link DataOutput#writeLong Int64}<sup>TableSize</sup>,{@link PackedInts PackedInts}</li>
- * <li>UncompressedNumerics --> {@link DataOutput#writeByte Byte}<sup>maxdoc</sup></li>
- * <li>Addresses --> {@link MonotonicBlockPackedWriter MonotonicBlockPackedInts(blockSize=4096)}</li>
- * <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}</li>
- * </ul>
- * <p>SortedSet entries store the list of ordinals in their BinaryData as a
- * sequences of increasing {@link DataOutput#writeVLong vLong}s, delta-encoded.</p>
- * </ol>
- * <p>
- * Limitations:
- * <ul>
- * <li> Binary doc values can be at most {@link #MAX_BINARY_FIELD_LENGTH} in length.
- * </ul>
* @deprecated Only for reading old 4.2 segments
*/
@Deprecated
public class Lucene42DocValuesFormat extends DocValuesFormat {
/** Maximum length for each binary doc values field. */
- public static final int MAX_BINARY_FIELD_LENGTH = (1 << 15) - 2;
+ static final int MAX_BINARY_FIELD_LENGTH = (1 << 15) - 2;
final float acceptableOverheadRatio;
@@ -162,7 +65,7 @@ public class Lucene42DocValuesFormat ext
}
@Override
- public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
+ public final DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
return new Lucene42DocValuesProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42DocValuesProducer.java Sun Oct 5 14:43:47 2014
@@ -66,9 +66,11 @@ import org.apache.lucene.util.packed.Mon
import org.apache.lucene.util.packed.PackedInts;
/**
- * Reader for {@link Lucene42DocValuesFormat}
+ * Reader for 4.2 docvalues
+ * @deprecated only for reading old 4.x segments
*/
-class Lucene42DocValuesProducer extends DocValuesProducer {
+@Deprecated
+final class Lucene42DocValuesProducer extends DocValuesProducer {
// metadata maps (just file pointers and minimal stuff)
private final Map<String,NumericEntry> numerics;
private final Map<String,BinaryEntry> binaries;
@@ -104,9 +106,34 @@ class Lucene42DocValuesProducer extends
static final int VERSION_GCD_COMPRESSION = 1;
static final int VERSION_CHECKSUM = 2;
static final int VERSION_CURRENT = VERSION_CHECKSUM;
+
+ private final boolean merging;
+
+ // clone for merge: when merging we don't do any instances.put()s
+ Lucene42DocValuesProducer(Lucene42DocValuesProducer original) throws IOException {
+ assert Thread.holdsLock(original);
+ numerics = original.numerics;
+ binaries = original.binaries;
+ fsts = original.fsts;
+ data = original.data.clone();
+ version = original.version;
+ numEntries = original.numEntries;
+
+ numericInstances.putAll(original.numericInstances);
+ binaryInstances.putAll(original.binaryInstances);
+ fstInstances.putAll(original.fstInstances);
+ numericInfo.putAll(original.numericInfo);
+ binaryInfo.putAll(original.binaryInfo);
+ addressInfo.putAll(original.addressInfo);
+
+ maxDoc = original.maxDoc;
+ ramBytesUsed = new AtomicLong(original.ramBytesUsed.get());
+ merging = true;
+ }
Lucene42DocValuesProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
maxDoc = state.segmentInfo.getDocCount();
+ merging = false;
String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
// read in the entries from the metadata file.
ChecksumIndexInput in = state.directory.openChecksumInput(metaName, state.context);
@@ -221,7 +248,9 @@ class Lucene42DocValuesProducer extends
NumericDocValues instance = numericInstances.get(field.name);
if (instance == null) {
instance = loadNumeric(field);
- numericInstances.put(field.name, instance);
+ if (!merging) {
+ numericInstances.put(field.name, instance);
+ }
}
return instance;
}
@@ -269,8 +298,10 @@ class Lucene42DocValuesProducer extends
final int formatID = data.readVInt();
final int bitsPerValue = data.readVInt();
final PackedInts.Reader ordsReader = PackedInts.getReaderNoHeader(data, PackedInts.Format.byId(formatID), entry.packedIntsVersion, maxDoc, bitsPerValue);
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
- numericInfo.put(field.name, ordsReader);
+ if (!merging) {
+ ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(decode) + ordsReader.ramBytesUsed());
+ numericInfo.put(field.name, ordsReader);
+ }
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -280,14 +311,18 @@ class Lucene42DocValuesProducer extends
case DELTA_COMPRESSED:
final int blockSize = data.readVInt();
final BlockPackedReader reader = new BlockPackedReader(data, entry.packedIntsVersion, blockSize, maxDoc, false);
- ramBytesUsed.addAndGet(reader.ramBytesUsed());
- numericInfo.put(field.name, reader);
+ if (!merging) {
+ ramBytesUsed.addAndGet(reader.ramBytesUsed());
+ numericInfo.put(field.name, reader);
+ }
return reader;
case UNCOMPRESSED:
final byte bytes[] = new byte[maxDoc];
data.readBytes(bytes, 0, bytes.length);
- ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
- numericInfo.put(field.name, Accountables.namedAccountable("byte array", maxDoc));
+ if (!merging) {
+ ramBytesUsed.addAndGet(RamUsageEstimator.sizeOf(bytes));
+ numericInfo.put(field.name, Accountables.namedAccountable("byte array", maxDoc));
+ }
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -299,8 +334,10 @@ class Lucene42DocValuesProducer extends
final long mult = data.readLong();
final int quotientBlockSize = data.readVInt();
final BlockPackedReader quotientReader = new BlockPackedReader(data, entry.packedIntsVersion, quotientBlockSize, maxDoc, false);
- ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
- numericInfo.put(field.name, quotientReader);
+ if (!merging) {
+ ramBytesUsed.addAndGet(quotientReader.ramBytesUsed());
+ numericInfo.put(field.name, quotientReader);
+ }
return new NumericDocValues() {
@Override
public long get(int docID) {
@@ -317,7 +354,9 @@ class Lucene42DocValuesProducer extends
BinaryDocValues instance = binaryInstances.get(field.name);
if (instance == null) {
instance = loadBinary(field);
- binaryInstances.put(field.name, instance);
+ if (!merging) {
+ binaryInstances.put(field.name, instance);
+ }
}
return instance;
}
@@ -328,10 +367,14 @@ class Lucene42DocValuesProducer extends
PagedBytes bytes = new PagedBytes(16);
bytes.copy(data, entry.numBytes);
final PagedBytes.Reader bytesReader = bytes.freeze(true);
- binaryInfo.put(field.name, bytesReader);
+ if (!merging) {
+ binaryInfo.put(field.name, bytesReader);
+ }
if (entry.minLength == entry.maxLength) {
final int fixedLength = entry.minLength;
- ramBytesUsed.addAndGet(bytesReader.ramBytesUsed());
+ if (!merging) {
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed());
+ }
return new BinaryDocValues() {
@Override
public BytesRef get(int docID) {
@@ -342,8 +385,10 @@ class Lucene42DocValuesProducer extends
};
} else {
final MonotonicBlockPackedReader addresses = MonotonicBlockPackedReader.of(data, entry.packedIntsVersion, entry.blockSize, maxDoc, false);
- addressInfo.put(field.name, addresses);
- ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addresses.ramBytesUsed());
+ if (!merging) {
+ addressInfo.put(field.name, addresses);
+ ramBytesUsed.addAndGet(bytesReader.ramBytesUsed() + addresses.ramBytesUsed());
+ }
return new BinaryDocValues() {
@Override
@@ -367,8 +412,10 @@ class Lucene42DocValuesProducer extends
if (instance == null) {
data.seek(entry.offset);
instance = new FST<>(data, PositiveIntOutputs.getSingleton());
- ramBytesUsed.addAndGet(instance.ramBytesUsed());
- fstInstances.put(field.name, instance);
+ if (!merging) {
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ fstInstances.put(field.name, instance);
+ }
}
}
final NumericDocValues docToOrd = getNumeric(field);
@@ -444,8 +491,10 @@ class Lucene42DocValuesProducer extends
if (instance == null) {
data.seek(entry.offset);
instance = new FST<>(data, PositiveIntOutputs.getSingleton());
- ramBytesUsed.addAndGet(instance.ramBytesUsed());
- fstInstances.put(field.name, instance);
+ if (!merging) {
+ ramBytesUsed.addAndGet(instance.ramBytesUsed());
+ fstInstances.put(field.name, instance);
+ }
}
}
final BinaryDocValues docToOrds = getBinary(field);
@@ -537,6 +586,11 @@ class Lucene42DocValuesProducer extends
}
@Override
+ public synchronized DocValuesProducer getMergeInstance() throws IOException {
+ return new Lucene42DocValuesProducer(this);
+ }
+
+ @Override
public void close() throws IOException {
data.close();
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosFormat.java Sun Oct 5 14:43:47 2014
@@ -19,70 +19,12 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
-import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldInfosFormat;
import org.apache.lucene.codecs.FieldInfosReader;
import org.apache.lucene.codecs.FieldInfosWriter;
-import org.apache.lucene.index.FieldInfo.DocValuesType; // javadoc
-import org.apache.lucene.store.DataOutput; // javadoc
/**
* Lucene 4.2 Field Infos format.
- * <p>
- * <p>Field names are stored in the field info file, with suffix <tt>.fnm</tt>.</p>
- * <p>FieldInfos (.fnm) --> Header,FieldsCount, <FieldName,FieldNumber,
- * FieldBits,DocValuesBits,Attributes> <sup>FieldsCount</sup></p>
- * <p>Data types:
- * <ul>
- * <li>Header --> {@link CodecUtil#checkHeader CodecHeader}</li>
- * <li>FieldsCount --> {@link DataOutput#writeVInt VInt}</li>
- * <li>FieldName --> {@link DataOutput#writeString String}</li>
- * <li>FieldBits, DocValuesBits --> {@link DataOutput#writeByte Byte}</li>
- * <li>FieldNumber --> {@link DataOutput#writeInt VInt}</li>
- * <li>Attributes --> {@link DataOutput#writeStringStringMap Map<String,String>}</li>
- * </ul>
- * </p>
- * Field Descriptions:
- * <ul>
- * <li>FieldsCount: the number of fields in this file.</li>
- * <li>FieldName: name of the field as a UTF-8 String.</li>
- * <li>FieldNumber: the field's number. Note that unlike previous versions of
- * Lucene, the fields are not numbered implicitly by their order in the
- * file, instead explicitly.</li>
- * <li>FieldBits: a byte containing field options.
- * <ul>
- * <li>The low-order bit is one for indexed fields, and zero for non-indexed
- * fields.</li>
- * <li>The second lowest-order bit is one for fields that have term vectors
- * stored, and zero for fields without term vectors.</li>
- * <li>If the third lowest order-bit is set (0x4), offsets are stored into
- * the postings list in addition to positions.</li>
- * <li>Fourth bit is unused.</li>
- * <li>If the fifth lowest-order bit is set (0x10), norms are omitted for the
- * indexed field.</li>
- * <li>If the sixth lowest-order bit is set (0x20), payloads are stored for the
- * indexed field.</li>
- * <li>If the seventh lowest-order bit is set (0x40), term frequencies and
- * positions omitted for the indexed field.</li>
- * <li>If the eighth lowest-order bit is set (0x80), positions are omitted for the
- * indexed field.</li>
- * </ul>
- * </li>
- * <li>DocValuesBits: a byte containing per-document value types. The type
- * recorded as two four-bit integers, with the high-order bits representing
- * <code>norms</code> options, and the low-order bits representing
- * {@code DocValues} options. Each four-bit integer can be decoded as such:
- * <ul>
- * <li>0: no DocValues for this field.</li>
- * <li>1: NumericDocValues. ({@link DocValuesType#NUMERIC})</li>
- * <li>2: BinaryDocValues. ({@code DocValuesType#BINARY})</li>
- * <li>3: SortedDocValues. ({@code DocValuesType#SORTED})</li>
- * </ul>
- * </li>
- * <li>Attributes: a key-value map of codec-private attributes.</li>
- * </ul>
- *
- * @lucene.experimental
* @deprecated Only for reading old 4.2-4.5 segments
*/
@Deprecated
@@ -94,7 +36,7 @@ public class Lucene42FieldInfosFormat ex
}
@Override
- public FieldInfosReader getFieldInfosReader() throws IOException {
+ public final FieldInfosReader getFieldInfosReader() throws IOException {
return reader;
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42FieldInfosReader.java Sun Oct 5 14:43:47 2014
@@ -38,9 +38,7 @@ import org.apache.lucene.util.IOUtils;
/**
* Lucene 4.2 FieldInfos reader.
*
- * @lucene.experimental
* @deprecated Only for reading old 4.2-4.5 segments
- * @see Lucene42FieldInfosFormat
*/
@Deprecated
final class Lucene42FieldInfosReader extends FieldInfosReader {
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsFormat.java Sun Oct 5 14:43:47 2014
@@ -19,8 +19,6 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
-import org.apache.lucene.codecs.DocValuesConsumer;
-import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.NormsProducer;
@@ -30,19 +28,9 @@ import org.apache.lucene.util.packed.Pac
/**
* Lucene 4.2 score normalization format.
- * <p>
- * NOTE: this uses the same format as {@link Lucene42DocValuesFormat}
- * Numeric DocValues, but with different file extensions, and passing
- * {@link PackedInts#FASTEST} for uncompressed encoding: trading off
- * space for performance.
- * <p>
- * Files:
- * <ul>
- * <li><tt>.nvd</tt>: DocValues data</li>
- * <li><tt>.nvm</tt>: DocValues metadata</li>
- * </ul>
- * @see Lucene42DocValuesFormat
+ * @deprecated only for reading old 4.x segments
*/
+@Deprecated
public class Lucene42NormsFormat extends NormsFormat {
final float acceptableOverheadRatio;
@@ -73,7 +61,7 @@ public class Lucene42NormsFormat extends
}
@Override
- public NormsProducer normsProducer(SegmentReadState state) throws IOException {
+ public final NormsProducer normsProducer(SegmentReadState state) throws IOException {
return new Lucene42NormsProducer(state, DATA_CODEC, DATA_EXTENSION, METADATA_CODEC, METADATA_EXTENSION);
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene42/Lucene42NormsProducer.java Sun Oct 5 14:43:47 2014
@@ -18,8 +18,8 @@ package org.apache.lucene.codecs.lucene4
*/
import java.io.IOException;
-import java.util.Collections;
+import org.apache.lucene.codecs.DocValuesProducer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.NumericDocValues;
@@ -28,12 +28,16 @@ import org.apache.lucene.util.Accountabl
/**
* Reads 4.2-4.8 norms.
- * Implemented the same as docvalues, but with a different filename.
* @deprecated Only for reading old segments
*/
@Deprecated
-class Lucene42NormsProducer extends NormsProducer {
- private final Lucene42DocValuesProducer impl;
+final class Lucene42NormsProducer extends NormsProducer {
+ private final DocValuesProducer impl;
+
+ // clone for merge
+ Lucene42NormsProducer(DocValuesProducer impl) throws IOException {
+ this.impl = impl.getMergeInstance();
+ }
Lucene42NormsProducer(SegmentReadState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
impl = new Lucene42DocValuesProducer(state, dataCodec, dataExtension, metaCodec, metaExtension);
@@ -65,6 +69,11 @@ class Lucene42NormsProducer extends Norm
}
@Override
+ public NormsProducer getMergeInstance() throws IOException {
+ return new Lucene42NormsProducer(impl);
+ }
+
+ @Override
public String toString() {
return getClass().getSimpleName() + "(" + impl + ")";
}
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45Codec.java Sun Oct 5 14:43:47 2014
@@ -20,9 +20,9 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.FieldInfosFormat;
-import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.LiveDocsFormat;
import org.apache.lucene.codecs.NormsConsumer;
import org.apache.lucene.codecs.NormsFormat;
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.SegmentInfoFormat;
import org.apache.lucene.codecs.StoredFieldsFormat;
import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene40.Lucene40CompoundFormat;
import org.apache.lucene.codecs.lucene40.Lucene40LiveDocsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40SegmentInfoFormat;
import org.apache.lucene.codecs.lucene41.Lucene41StoredFieldsFormat;
@@ -41,19 +42,9 @@ import org.apache.lucene.codecs.perfield
import org.apache.lucene.index.SegmentWriteState;
/**
- * Implements the Lucene 4.5 index format, with configurable per-field postings
- * and docvalues formats.
- * <p>
- * If you want to reuse functionality of this codec in another codec, extend
- * {@link FilterCodec}.
- *
- * @see org.apache.lucene.codecs.lucene45 package documentation for file format details.
- * @lucene.experimental
+ * Implements the Lucene 4.5 index format
* @deprecated Only for reading old 4.3-4.5 segments
*/
-// NOTE: if we make largish changes in a minor release, easier to just make Lucene46Codec or whatever
-// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader
-// (it writes a minor version, etc).
@Deprecated
public class Lucene45Codec extends Codec {
private final StoredFieldsFormat fieldsFormat = new Lucene41StoredFieldsFormat();
@@ -61,6 +52,7 @@ public class Lucene45Codec extends Codec
private final FieldInfosFormat fieldInfosFormat = new Lucene42FieldInfosFormat();
private final SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat();
private final LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene40CompoundFormat();
private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
@Override
@@ -83,12 +75,12 @@ public class Lucene45Codec extends Codec
}
@Override
- public final StoredFieldsFormat storedFieldsFormat() {
+ public StoredFieldsFormat storedFieldsFormat() {
return fieldsFormat;
}
@Override
- public final TermVectorsFormat termVectorsFormat() {
+ public TermVectorsFormat termVectorsFormat() {
return vectorsFormat;
}
@@ -111,6 +103,11 @@ public class Lucene45Codec extends Codec
public final LiveDocsFormat liveDocsFormat() {
return liveDocsFormat;
}
+
+ @Override
+ public CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
/** Returns the postings format that should be used for writing
* new segments of <code>field</code>.
Modified: lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java?rev=1629501&r1=1629500&r2=1629501&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/backward-codecs/src/java/org/apache/lucene/codecs/lucene45/Lucene45DocValuesConsumer.java Sun Oct 5 14:43:47 2014
@@ -39,7 +39,11 @@ import org.apache.lucene.util.packed.Blo
import org.apache.lucene.util.packed.MonotonicBlockPackedWriter;
import org.apache.lucene.util.packed.PackedInts;
-/** writer for {@link Lucene45DocValuesFormat} */
+/**
+ * writer for 4.5 docvalues format
+ * @deprecated only for old 4.x segments
+ */
+@Deprecated
class Lucene45DocValuesConsumer extends DocValuesConsumer implements Closeable {
static final int BLOCK_SIZE = 16384;