You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/04/04 17:19:41 UTC
[35/62] [abbrv] [partial] lucenenet git commit: Renamed
Lucene.Net.Core folder Lucene.Net because the dotnet.exe pack command doesn't
allow creating a NuGet package with a different name than its folder. Working
around it with the script was much more co
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Lucene41/ForUtil.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Lucene41/ForUtil.cs b/src/Lucene.Net.Core/Codecs/Lucene41/ForUtil.cs
deleted file mode 100644
index 3488cb3..0000000
--- a/src/Lucene.Net.Core/Codecs/Lucene41/ForUtil.cs
+++ /dev/null
@@ -1,265 +0,0 @@
-using Lucene.Net.Store;
-using Lucene.Net.Support;
-using Lucene.Net.Util.Packed;
-using System;
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.Lucene41
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- /*
- using DataInput = Lucene.Net.Store.DataInput;
- using DataOutput = Lucene.Net.Store.DataOutput;
- using IndexInput = Lucene.Net.Store.IndexInput;
- using IndexOutput = Lucene.Net.Store.IndexOutput;
- using Decoder = Lucene.Net.Util.Packed.PackedInts.Decoder;
- using FormatAndBits = Lucene.Net.Util.Packed.PackedInts.FormatAndBits;
- using PackedInts = Lucene.Net.Util.Packed.PackedInts;
- */
-
- /// <summary>
- /// Encode all values in normal area with fixed bit width,
- /// which is determined by the max value in this block.
- /// </summary>
- internal sealed class ForUtil
- {
- /// <summary>
- /// Special number of bits per value used whenever all values to encode are equal.
- /// </summary>
- private static readonly int ALL_VALUES_EQUAL = 0;
-
- /// <summary>
- /// Upper limit of the number of bytes that might be required to stored
- /// <code>BLOCK_SIZE</code> encoded values.
- /// </summary>
- public static readonly int MAX_ENCODED_SIZE = Lucene41PostingsFormat.BLOCK_SIZE * 4;
-
- /// <summary>
- /// Upper limit of the number of values that might be decoded in a single call to
- /// <seealso cref="#readBlock(IndexInput, byte[], int[])"/>. Although values after
- /// <code>BLOCK_SIZE</code> are garbage, it is necessary to allocate value buffers
- /// whose size is >= MAX_DATA_SIZE to avoid <seealso cref="ArrayIndexOutOfBoundsException"/>s.
- /// </summary>
- public static readonly int MAX_DATA_SIZE;
-
- static ForUtil()
- {
- int maxDataSize = 0;
- for (int version = PackedInt32s.VERSION_START; version <= PackedInt32s.VERSION_CURRENT; version++)
- {
- foreach (PackedInt32s.Format format in PackedInt32s.Format.Values()/* Enum.GetValues(typeof(PackedInts.Format))*/)
- {
- for (int bpv = 1; bpv <= 32; ++bpv)
- {
- if (!format.IsSupported(bpv))
- {
- continue;
- }
- PackedInt32s.IDecoder decoder = PackedInt32s.GetDecoder(format, version, bpv);
- int iterations = ComputeIterations(decoder);
- maxDataSize = Math.Max(maxDataSize, iterations * decoder.ByteValueCount);
- }
- }
- }
- MAX_DATA_SIZE = maxDataSize;
- }
-
- /// <summary>
- /// Compute the number of iterations required to decode <code>BLOCK_SIZE</code>
- /// values with the provided <seealso cref="Decoder"/>.
- /// </summary>
- private static int ComputeIterations(PackedInt32s.IDecoder decoder)
- {
- return (int)Math.Ceiling((float)Lucene41PostingsFormat.BLOCK_SIZE / decoder.ByteValueCount);
- }
-
- /// <summary>
- /// Compute the number of bytes required to encode a block of values that require
- /// <code>bitsPerValue</code> bits per value with format <code>format</code>.
- /// </summary>
- private static int EncodedSize(PackedInt32s.Format format, int packedIntsVersion, int bitsPerValue)
- {
- long byteCount = format.ByteCount(packedIntsVersion, Lucene41PostingsFormat.BLOCK_SIZE, bitsPerValue);
- Debug.Assert(byteCount >= 0 && byteCount <= int.MaxValue, byteCount.ToString());
- return (int)byteCount;
- }
-
- private readonly int[] encodedSizes;
- private readonly PackedInt32s.IEncoder[] encoders;
- private readonly PackedInt32s.IDecoder[] decoders;
- private readonly int[] iterations;
-
- /// <summary>
- /// Create a new <seealso cref="ForUtil"/> instance and save state into <code>out</code>.
- /// </summary>
- internal ForUtil(float acceptableOverheadRatio, DataOutput @out)
- {
- @out.WriteVInt32(PackedInt32s.VERSION_CURRENT);
- encodedSizes = new int[33];
- encoders = new PackedInt32s.IEncoder[33];
- decoders = new PackedInt32s.IDecoder[33];
- iterations = new int[33];
-
- for (int bpv = 1; bpv <= 32; ++bpv)
- {
- PackedInt32s.FormatAndBits formatAndBits = PackedInt32s.FastestFormatAndBits(Lucene41PostingsFormat.BLOCK_SIZE, bpv, acceptableOverheadRatio);
- Debug.Assert(formatAndBits.Format.IsSupported(formatAndBits.BitsPerValue));
- Debug.Assert(formatAndBits.BitsPerValue <= 32);
- encodedSizes[bpv] = EncodedSize(formatAndBits.Format, PackedInt32s.VERSION_CURRENT, formatAndBits.BitsPerValue);
- encoders[bpv] = PackedInt32s.GetEncoder(formatAndBits.Format, PackedInt32s.VERSION_CURRENT, formatAndBits.BitsPerValue);
- decoders[bpv] = PackedInt32s.GetDecoder(formatAndBits.Format, PackedInt32s.VERSION_CURRENT, formatAndBits.BitsPerValue);
- iterations[bpv] = ComputeIterations(decoders[bpv]);
-
- @out.WriteVInt32(formatAndBits.Format.Id << 5 | (formatAndBits.BitsPerValue - 1));
- }
- }
-
- /// <summary>
- /// Restore a <seealso cref="ForUtil"/> from a <seealso cref="DataInput"/>.
- /// </summary>
- internal ForUtil(DataInput @in)
- {
- int packedIntsVersion = @in.ReadVInt32();
- PackedInt32s.CheckVersion(packedIntsVersion);
- encodedSizes = new int[33];
- encoders = new PackedInt32s.IEncoder[33];
- decoders = new PackedInt32s.IDecoder[33];
- iterations = new int[33];
-
- for (int bpv = 1; bpv <= 32; ++bpv)
- {
- var code = @in.ReadVInt32();
- var formatId = (int)((uint)code >> 5);
- var bitsPerValue = (code & 31) + 1;
-
- PackedInt32s.Format format = PackedInt32s.Format.ById(formatId);
- Debug.Assert(format.IsSupported(bitsPerValue));
- encodedSizes[bpv] = EncodedSize(format, packedIntsVersion, bitsPerValue);
- encoders[bpv] = PackedInt32s.GetEncoder(format, packedIntsVersion, bitsPerValue);
- decoders[bpv] = PackedInt32s.GetDecoder(format, packedIntsVersion, bitsPerValue);
- iterations[bpv] = ComputeIterations(decoders[bpv]);
- }
- }
-
- /// <summary>
- /// Write a block of data (<code>For</code> format).
- /// </summary>
- /// <param name="data"> the data to write </param>
- /// <param name="encoded"> a buffer to use to encode data </param>
- /// <param name="out"> the destination output </param>
- /// <exception cref="IOException"> If there is a low-level I/O error </exception>
- internal void WriteBlock(int[] data, byte[] encoded, IndexOutput @out)
- {
- if (IsAllEqual(data))
- {
- @out.WriteByte((byte)(sbyte)ALL_VALUES_EQUAL);
- @out.WriteVInt32(data[0]);
- return;
- }
-
- int numBits = BitsRequired(data);
- Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
- PackedInt32s.IEncoder encoder = encoders[numBits];
- int iters = iterations[numBits];
- Debug.Assert(iters * encoder.ByteValueCount >= Lucene41PostingsFormat.BLOCK_SIZE);
- int encodedSize = encodedSizes[numBits];
- Debug.Assert(iters * encoder.ByteBlockCount >= encodedSize);
-
- @out.WriteByte((byte)numBits);
-
- encoder.Encode(data, 0, encoded, 0, iters);
- @out.WriteBytes(encoded, encodedSize);
- }
-
- /// <summary>
- /// Read the next block of data (<code>For</code> format).
- /// </summary>
- /// <param name="in"> the input to use to read data </param>
- /// <param name="encoded"> a buffer that can be used to store encoded data </param>
- /// <param name="decoded"> where to write decoded data </param>
- /// <exception cref="IOException"> If there is a low-level I/O error </exception>
- internal void ReadBlock(IndexInput @in, byte[] encoded, int[] decoded)
- {
- int numBits = @in.ReadByte();
- Debug.Assert(numBits <= 32, numBits.ToString());
-
- if (numBits == ALL_VALUES_EQUAL)
- {
- int value = @in.ReadVInt32();
- Arrays.Fill(decoded, 0, Lucene41PostingsFormat.BLOCK_SIZE, value);
- return;
- }
-
- int encodedSize = encodedSizes[numBits];
- @in.ReadBytes(encoded, 0, encodedSize);
-
- PackedInt32s.IDecoder decoder = decoders[numBits];
- int iters = iterations[numBits];
- Debug.Assert(iters * decoder.ByteValueCount >= Lucene41PostingsFormat.BLOCK_SIZE);
-
- decoder.Decode(encoded, 0, decoded, 0, iters);
- }
-
- /// <summary>
- /// Skip the next block of data.
- /// </summary>
- /// <param name="in"> the input where to read data </param>
- /// <exception cref="IOException"> If there is a low-level I/O error </exception>
- internal void SkipBlock(IndexInput @in)
- {
- int numBits = @in.ReadByte();
- if (numBits == ALL_VALUES_EQUAL)
- {
- @in.ReadVInt32();
- return;
- }
- Debug.Assert(numBits > 0 && numBits <= 32, numBits.ToString());
- int encodedSize = encodedSizes[numBits];
- @in.Seek(@in.GetFilePointer() + encodedSize);
- }
-
- private static bool IsAllEqual(int[] data)
- {
- int v = data[0];
- for (int i = 1; i < Lucene41PostingsFormat.BLOCK_SIZE; ++i)
- {
- if (data[i] != v)
- {
- return false;
- }
- }
- return true;
- }
-
- /// <summary>
- /// Compute the number of bits required to serialize any of the longs in
- /// <code>data</code>.
- /// </summary>
- private static int BitsRequired(int[] data)
- {
- long or = 0;
- for (int i = 0; i < Lucene41PostingsFormat.BLOCK_SIZE; ++i)
- {
- Debug.Assert(data[i] >= 0);
- or |= (uint)data[i];
- }
- return PackedInt32s.BitsRequired(or);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41Codec.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41Codec.cs b/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41Codec.cs
deleted file mode 100644
index c59c251..0000000
--- a/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41Codec.cs
+++ /dev/null
@@ -1,150 +0,0 @@
-using System;
-
-namespace Lucene.Net.Codecs.Lucene41
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- using CompressingStoredFieldsFormat = Lucene.Net.Codecs.Compressing.CompressingStoredFieldsFormat;
- using CompressionMode = Lucene.Net.Codecs.Compressing.CompressionMode;
- using Directory = Lucene.Net.Store.Directory;
- using IOContext = Lucene.Net.Store.IOContext;
- using Lucene40DocValuesFormat = Lucene.Net.Codecs.Lucene40.Lucene40DocValuesFormat;
- using Lucene40FieldInfosFormat = Lucene.Net.Codecs.Lucene40.Lucene40FieldInfosFormat;
- using Lucene40LiveDocsFormat = Lucene.Net.Codecs.Lucene40.Lucene40LiveDocsFormat;
- using Lucene40NormsFormat = Lucene.Net.Codecs.Lucene40.Lucene40NormsFormat;
- using Lucene40SegmentInfoFormat = Lucene.Net.Codecs.Lucene40.Lucene40SegmentInfoFormat;
- using Lucene40TermVectorsFormat = Lucene.Net.Codecs.Lucene40.Lucene40TermVectorsFormat;
- using PerFieldPostingsFormat = Lucene.Net.Codecs.PerField.PerFieldPostingsFormat;
- using SegmentInfo = Lucene.Net.Index.SegmentInfo;
-
- /// <summary>
- /// Implements the Lucene 4.1 index format, with configurable per-field postings formats.
- /// <p>
- /// If you want to reuse functionality of this codec in another codec, extend
- /// <seealso cref="FilterCodec"/>.
- /// </summary>
- /// <seealso cref= Lucene.Net.Codecs.Lucene41 package documentation for file format details. </seealso>
- /// @deprecated Only for reading old 4.0 segments
- /// @lucene.experimental
- [Obsolete("Only for reading old 4.0 segments")]
- [CodecName("Lucene41")] // LUCENENET specific - using CodecName attribute to ensure the default name passed from subclasses is the same as this class name
- public class Lucene41Codec : Codec
- {
- // TODO: slightly evil
- private readonly StoredFieldsFormat fieldsFormat = new CompressingStoredFieldsFormatAnonymousInnerClassHelper("Lucene41StoredFields", CompressionMode.FAST, 1 << 14);
-
- private class CompressingStoredFieldsFormatAnonymousInnerClassHelper : CompressingStoredFieldsFormat
- {
- public CompressingStoredFieldsFormatAnonymousInnerClassHelper(string formatName, CompressionMode compressionMode, int chunkSize)
- : base(formatName, compressionMode, chunkSize)
- {
- }
-
- public override StoredFieldsWriter FieldsWriter(Directory directory, SegmentInfo si, IOContext context)
- {
- throw new System.NotSupportedException("this codec can only be used for reading");
- }
- }
-
- private readonly TermVectorsFormat vectorsFormat = new Lucene40TermVectorsFormat();
- private readonly FieldInfosFormat fieldInfosFormat = new Lucene40FieldInfosFormat();
- private readonly SegmentInfoFormat infosFormat = new Lucene40SegmentInfoFormat();
- private readonly LiveDocsFormat liveDocsFormat = new Lucene40LiveDocsFormat();
-
- private readonly PostingsFormat postingsFormat;
-
- private class PerFieldPostingsFormatAnonymousInnerClassHelper : PerFieldPostingsFormat
- {
- private readonly Lucene41Codec outerInstance;
-
- public PerFieldPostingsFormatAnonymousInnerClassHelper(Lucene41Codec outerInstance)
- {
- this.outerInstance = outerInstance;
- }
-
- public override PostingsFormat GetPostingsFormatForField(string field)
- {
- return outerInstance.GetPostingsFormatForField(field);
- }
- }
-
- /// <summary>
- /// Sole constructor. </summary>
- public Lucene41Codec()
- : base()
- {
- postingsFormat = new PerFieldPostingsFormatAnonymousInnerClassHelper(this);
- }
-
- // TODO: slightly evil
- public override StoredFieldsFormat StoredFieldsFormat
- {
- get { return fieldsFormat; }
- }
-
- public override sealed TermVectorsFormat TermVectorsFormat
- {
- get { return vectorsFormat; }
- }
-
- public override sealed PostingsFormat PostingsFormat
- {
- get { return postingsFormat; }
- }
-
- public override FieldInfosFormat FieldInfosFormat
- {
- get { return fieldInfosFormat; }
- }
-
- public override SegmentInfoFormat SegmentInfoFormat
- {
- get { return infosFormat; }
- }
-
- public override sealed LiveDocsFormat LiveDocsFormat
- {
- get { return liveDocsFormat; }
- }
-
- /// <summary>
- /// Returns the postings format that should be used for writing
- /// new segments of <code>field</code>.
- ///
- /// The default implementation always returns "Lucene41"
- /// </summary>
- public virtual PostingsFormat GetPostingsFormatForField(string field)
- {
- return defaultFormat;
- }
-
- public override DocValuesFormat DocValuesFormat
- {
- get { return dvFormat; }
- }
-
- private readonly PostingsFormat defaultFormat = Codecs.PostingsFormat.ForName("Lucene41");
- private readonly DocValuesFormat dvFormat = new Lucene40DocValuesFormat();
- private readonly NormsFormat normsFormat = new Lucene40NormsFormat();
-
- public override NormsFormat NormsFormat
- {
- get { return normsFormat; }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsBaseFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsBaseFormat.cs b/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsBaseFormat.cs
deleted file mode 100644
index 6c27407..0000000
--- a/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsBaseFormat.cs
+++ /dev/null
@@ -1,50 +0,0 @@
-namespace Lucene.Net.Codecs.Lucene41
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- using SegmentReadState = Lucene.Net.Index.SegmentReadState;
- using SegmentWriteState = Lucene.Net.Index.SegmentWriteState;
-
- /// <summary>
- /// Provides a <seealso cref="PostingsReaderBase"/> and {@link
- /// PostingsWriterBase}.
- ///
- /// @lucene.experimental
- /// </summary>
-
- // TODO: should these also be named / looked up via SPI?
- public sealed class Lucene41PostingsBaseFormat : PostingsBaseFormat
- {
- /// <summary>
- /// Sole constructor. </summary>
- public Lucene41PostingsBaseFormat()
- : base("Lucene41")
- {
- }
-
- public override PostingsReaderBase PostingsReaderBase(SegmentReadState state)
- {
- return new Lucene41PostingsReader(state.Directory, state.FieldInfos, state.SegmentInfo, state.Context, state.SegmentSuffix);
- }
-
- public override PostingsWriterBase PostingsWriterBase(SegmentWriteState state)
- {
- return new Lucene41PostingsWriter(state);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/a5dc68d0/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsFormat.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsFormat.cs b/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsFormat.cs
deleted file mode 100644
index 8b0c6ea..0000000
--- a/src/Lucene.Net.Core/Codecs/Lucene41/Lucene41PostingsFormat.cs
+++ /dev/null
@@ -1,444 +0,0 @@
-using System.Diagnostics;
-
-namespace Lucene.Net.Codecs.Lucene41
-{
- /*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- using IOUtils = Lucene.Net.Util.IOUtils;
- using SegmentReadState = Lucene.Net.Index.SegmentReadState;
- using SegmentWriteState = Lucene.Net.Index.SegmentWriteState;
-
- /// <summary>
- /// Lucene 4.1 postings format, which encodes postings in packed integer blocks
- /// for fast decode.
- ///
- /// <p><b>NOTE</b>: this format is still experimental and
- /// subject to change without backwards compatibility.
- ///
- /// <p>
- /// Basic idea:
- /// <ul>
- /// <li>
- /// <b>Packed Blocks and VInt Blocks</b>:
- /// <p>In packed blocks, integers are encoded with the same bit width (<seealso cref="PackedInts packed format"/>):
- /// the block size (i.e. number of integers inside block) is fixed (currently 128). Additionally blocks
- /// that are all the same value are encoded in an optimized way.</p>
- /// <p>In VInt blocks, integers are encoded as <seealso cref="DataOutput#writeVInt VInt"/>:
- /// the block size is variable.</p>
- /// </li>
- ///
- /// <li>
- /// <b>Block structure</b>:
- /// <p>When the postings are long enough, Lucene41PostingsFormat will try to encode most integer data
- /// as a packed block.</p>
- /// <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as two packed
- /// blocks, while the remaining 3 are encoded as one VInt block. </p>
- /// <p>Different kinds of data are always encoded separately into different packed blocks, but may
- /// possibly be interleaved into the same VInt block. </p>
- /// <p>this strategy is applied to pairs:
- /// <document number, frequency>,
- /// <position, payload length>,
- /// <position, offset start, offset length>, and
- /// <position, payload length, offsetstart, offset length>.</p>
- /// </li>
- ///
- /// <li>
- /// <b>Skipdata settings</b>:
- /// <p>The structure of skip table is quite similar to previous version of Lucene. Skip interval is the
- /// same as block size, and each skip entry points to the beginning of each block. However, for
- /// the first block, skip data is omitted.</p>
- /// </li>
- ///
- /// <li>
- /// <b>Positions, Payloads, and Offsets</b>:
- /// <p>A position is an integer indicating where the term occurs within one document.
- /// A payload is a blob of metadata associated with current position.
- /// An offset is a pair of integers indicating the tokenized start/end offsets for given term
- /// in current position: it is essentially a specialized payload. </p>
- /// <p>When payloads and offsets are not omitted, numPositions==numPayloads==numOffsets (assuming a
- /// null payload contributes one count). As mentioned in block structure, it is possible to encode
- /// these three either combined or separately.
- /// <p>In all cases, payloads and offsets are stored together. When encoded as a packed block,
- /// position data is separated out as .pos, while payloads and offsets are encoded in .pay (payload
- /// metadata will also be stored directly in .pay). When encoded as VInt blocks, all these three are
- /// stored interleaved into the .pos (so is payload metadata).</p>
- /// <p>With this strategy, the majority of payload and offset data will be outside .pos file.
- /// So for queries that require only position data, running on a full index with payloads and offsets,
- /// this reduces disk pre-fetches.</p>
- /// </li>
- /// </ul>
- /// </p>
- ///
- /// <p>
- /// Files and detailed format:
- /// <ul>
- /// <li><tt>.tim</tt>: <a href="#Termdictionary">Term Dictionary</a></li>
- /// <li><tt>.tip</tt>: <a href="#Termindex">Term Index</a></li>
- /// <li><tt>.doc</tt>: <a href="#Frequencies">Frequencies and Skip Data</a></li>
- /// <li><tt>.pos</tt>: <a href="#Positions">Positions</a></li>
- /// <li><tt>.pay</tt>: <a href="#Payloads">Payloads and Offsets</a></li>
- /// </ul>
- /// </p>
- ///
- /// <a name="Termdictionary" id="Termdictionary"></a>
- /// <dl>
- /// <dd>
- /// <b>Term Dictionary</b>
- ///
- /// <p>The .tim file contains the list of terms in each
- /// field along with per-term statistics (such as docfreq)
- /// and pointers to the frequencies, positions, payload and
- /// skip data in the .doc, .pos, and .pay files.
- /// See <seealso cref="BlockTreeTermsWriter"/> for more details on the format.
- /// </p>
- ///
- /// <p>NOTE: The term dictionary can plug into different postings implementations:
- /// the postings writer/reader are actually responsible for encoding
- /// and decoding the PostingsHeader and TermMetadata sections described here:</p>
- ///
- /// <ul>
- /// <li>PostingsHeader --> Header, PackedBlockSize</li>
- /// <li>TermMetadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?, PayFPDelta?,
- /// SkipFPDelta?</li>
- /// <li>Header, --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
- /// <li>PackedBlockSize, SingletonDocID --> <seealso cref="DataOutput#writeVInt VInt"/></li>
- /// <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> <seealso cref="DataOutput#writeVLong VLong"/></li>
- /// <li>Footer --> <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
- /// </ul>
- /// <p>Notes:</p>
- /// <ul>
- /// <li>Header is a <seealso cref="CodecUtil#writeHeader CodecHeader"/> storing the version information
- /// for the postings.</li>
- /// <li>PackedBlockSize is the fixed block size for packed blocks. In packed block, bit width is
- /// determined by the largest integer. Smaller block size result in smaller variance among width
- /// of integers hence smaller indexes. Larger block size result in more efficient bulk i/o hence
- /// better acceleration. this value should always be a multiple of 64, currently fixed as 128 as
- /// a tradeoff. It is also the skip interval used to accelerate <seealso cref="DocsEnum#advance(int)"/>.
- /// <li>DocFPDelta determines the position of this term's TermFreqs within the .doc file.
- /// In particular, it is the difference of file offset between this term's
- /// data and previous term's data (or zero, for the first term in the block).On disk it is
- /// stored as the difference from previous value in sequence. </li>
- /// <li>PosFPDelta determines the position of this term's TermPositions within the .pos file.
- /// While PayFPDelta determines the position of this term's <TermPayloads, TermOffsets?> within
- /// the .pay file. Similar to DocFPDelta, it is the difference between two file positions (or
- /// neglected, for fields that omit payloads and offsets).</li>
- /// <li>PosVIntBlockFPDelta determines the position of this term's last TermPosition in last pos packed
- /// block within the .pos file. It is synonym for PayVIntBlockFPDelta or OffsetVIntBlockFPDelta.
- /// this is actually used to indicate whether it is necessary to load following
- /// payloads and offsets from .pos instead of .pay. Every time a new block of positions are to be
- /// loaded, the PostingsReader will use this value to check whether current block is packed format
- /// or VInt. When packed format, payloads and offsets are fetched from .pay, otherwise from .pos.
- /// (this value is neglected when total number of positions i.e. totalTermFreq is less or equal
- /// to PackedBlockSize).
- /// <li>SkipFPDelta determines the position of this term's SkipData within the .doc
- /// file. In particular, it is the length of the TermFreq data.
- /// SkipDelta is only stored if DocFreq is not smaller than SkipMinimum
- /// (i.e. 128 in Lucene41PostingsFormat).</li>
- /// <li>SingletonDocID is an optimization when a term only appears in one document. In this case, instead
- /// of writing a file pointer to the .doc file (DocFPDelta), and then a VIntBlock at that location, the
- /// single document ID is written to the term dictionary.</li>
- /// </ul>
- /// </dd>
- /// </dl>
- ///
- /// <a name="Termindex" id="Termindex"></a>
- /// <dl>
- /// <dd>
- /// <b>Term Index</b>
- /// <p>The .tip file contains an index into the term dictionary, so that it can be
- /// accessed randomly. See <seealso cref="BlockTreeTermsWriter"/> for more details on the format.</p>
- /// </dd>
- /// </dl>
- ///
- ///
- /// <a name="Frequencies" id="Frequencies"></a>
- /// <dl>
- /// <dd>
- /// <b>Frequencies and Skip Data</b>
- ///
- /// <p>The .doc file contains the lists of documents which contain each term, along
- /// with the frequency of the term in that document (except when frequencies are
- /// omitted: <seealso cref="IndexOptions#DOCS_ONLY"/>). It also saves skip data to the beginning of
- /// each packed or VInt block, when the length of document list is larger than packed block size.</p>
- ///
- /// <ul>
- /// <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup>, Footer</li>
- /// <li>Header --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
- /// <li>TermFreqs --> <PackedBlock> <sup>PackedDocBlockNum</sup>,
- /// VIntBlock? </li>
- /// <li>PackedBlock --> PackedDocDeltaBlock, PackedFreqBlock?
- /// <li>VIntBlock --> <DocDelta[, Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
- /// <li>SkipData --> <<SkipLevelLength, SkipLevel>
- /// <sup>NumSkipLevels-1</sup>, SkipLevel>, SkipDatum?</li>
- /// <li>SkipLevel --> <SkipDatum> <sup>TrimmedDocFreq/(PackedBlockSize^(Level + 1))</sup></li>
- /// <li>SkipDatum --> DocSkip, DocFPSkip, <PosFPSkip, PosBlockOffset, PayLength?,
- /// PayFPSkip?>?, SkipChildLevelPointer?</li>
- /// <li>PackedDocDeltaBlock, PackedFreqBlock --> <seealso cref="PackedInts PackedInts"/></li>
- /// <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto, PayFPSkip
- /// -->
- /// <seealso cref="DataOutput#writeVInt VInt"/></li>
- /// <li>SkipChildLevelPointer --> <seealso cref="DataOutput#writeVLong VLong"/></li>
- /// <li>Footer --> <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
- /// </ul>
- /// <p>Notes:</p>
- /// <ul>
- /// <li>PackedDocDeltaBlock is theoretically generated from two steps:
- /// <ol>
- /// <li>Calculate the difference between each document number and previous one,
- /// and get a d-gaps list (for the first document, use absolute value); </li>
- /// <li>For those d-gaps from first one to PackedDocBlockNum*PackedBlockSize<sup>th</sup>,
- /// separately encode as packed blocks.</li>
- /// </ol>
- /// If frequencies are not omitted, PackedFreqBlock will be generated without d-gap step.
- /// </li>
- /// <li>VIntBlock stores remaining d-gaps (along with frequencies when possible) with a format
- /// that encodes DocDelta and Freq:
- /// <p>DocDelta: if frequencies are indexed, this determines both the document
- /// number and the frequency. In particular, DocDelta/2 is the difference between
- /// this document number and the previous document number (or zero when this is the
- /// first document in a TermFreqs). When DocDelta is odd, the frequency is one.
- /// When DocDelta is even, the frequency is read as another VInt. If frequencies
- /// are omitted, DocDelta contains the gap (not multiplied by 2) between document
- /// numbers and no frequency information is stored.</p>
- /// <p>For example, the TermFreqs for a term which occurs once in document seven
- /// and three times in document eleven, with frequencies indexed, would be the
- /// following sequence of VInts:</p>
- /// <p>15, 8, 3</p>
- /// <p>If frequencies were omitted (<seealso cref="IndexOptions#DOCS_ONLY"/>) it would be this
- /// sequence of VInts instead:</p>
- /// <p>7,4</p>
- /// </li>
- /// <li>PackedDocBlockNum is the number of packed blocks for current term's docids or frequencies.
- /// In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize) </li>
- /// <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq.
- /// We use this trick since the definition of skip entry is a little different from base interface.
- /// In <seealso cref="MultiLevelSkipListWriter"/>, skip data is assumed to be saved for
- /// skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list. However,
- /// in Lucene41PostingsFormat, the skip data is saved for skipInterval+1<sup>th</sup>,
- /// 2*skipInterval+1<sup>th</sup> ... posting (skipInterval==PackedBlockSize in this case).
- /// When DocFreq is multiple of PackedBlockSize, MultiLevelSkipListWriter will expect one
- /// more skip data than Lucene41SkipWriter. </li>
- /// <li>SkipDatum is the metadata of one skip entry.
- /// For the first block (no matter packed or VInt), it is omitted.</li>
- /// <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document number in
- /// the postings (i.e. last document number in each packed block). On disk it is stored as the
- /// difference from previous value in the sequence. </li>
- /// <li>DocFPSkip records the file offsets of each block (excluding )posting at
- /// PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.
- /// The file offsets are relative to the start of current term's TermFreqs.
- /// On disk it is also stored as the difference from previous SkipDatum in the sequence.</li>
- /// <li>Since positions and payloads are also block encoded, the skip should skip to related block first,
- /// then fetch the values according to in-block offset. PosFPSkip and PayFPSkip record the file
- /// offsets of related block in .pos and .pay, respectively. While PosBlockOffset indicates
- /// which value to fetch inside the related block (PayBlockOffset is unnecessary since it is always
- /// equal to PosBlockOffset). Same as DocFPSkip, the file offsets are relative to the start of
- /// current term's TermFreqs, and stored as a difference sequence.</li>
- /// <li>PayByteUpto indicates the start offset of the current payload. It is equivalent to
- /// the sum of the payload lengths in the current block up to PosBlockOffset</li>
- /// </ul>
- /// </dd>
- /// </dl>
- ///
- /// <a name="Positions" id="Positions"></a>
- /// <dl>
- /// <dd>
- /// <b>Positions</b>
- /// <p>The .pos file contains the lists of positions that each term occurs at within documents. It also
- /// sometimes stores part of payloads and offsets for speedup.</p>
- /// <ul>
- /// <li>PosFile(.pos) --> Header, <TermPositions> <sup>TermCount</sup>, Footer</li>
- /// <li>Header --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
- /// <li>TermPositions --> <PackedPosDeltaBlock> <sup>PackedPosBlockNum</sup>,
- /// VIntBlock? </li>
- /// <li>VIntBlock --> <PositionDelta[, PayloadLength?], PayloadData?,
- /// OffsetDelta?, OffsetLength?><sup>PosVIntCount</sup>
- /// <li>PackedPosDeltaBlock --> <seealso cref="PackedInts PackedInts"/></li>
- /// <li>PositionDelta, OffsetDelta, OffsetLength -->
- /// <seealso cref="DataOutput#writeVInt VInt"/></li>
- /// <li>PayloadData --> <seealso cref="DataOutput#writeByte byte"/><sup>PayLength</sup></li>
- /// <li>Footer --> <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
- /// </ul>
- /// <p>Notes:</p>
- /// <ul>
- /// <li>TermPositions are order by term (terms are implicit, from the term dictionary), and position
- /// values for each term document pair are incremental, and ordered by document number.</li>
- /// <li>PackedPosBlockNum is the number of packed blocks for current term's positions, payloads or offsets.
- /// In particular, PackedPosBlockNum = floor(totalTermFreq/PackedBlockSize) </li>
- /// <li>PosVIntCount is the number of positions encoded as VInt format. In particular,
- /// PosVIntCount = totalTermFreq - PackedPosBlockNum*PackedBlockSize</li>
- /// <li>The procedure how PackedPosDeltaBlock is generated is the same as PackedDocDeltaBlock
- /// in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.</li>
- /// <li>PositionDelta is, if payloads are disabled for the term's field, the
- /// difference between the position of the current occurrence in the document and
- /// the previous occurrence (or zero, if this is the first occurrence in this
- /// document). If payloads are enabled for the term's field, then PositionDelta/2
- /// is the difference between the current and the previous position. If payloads
- /// are enabled and PositionDelta is odd, then PayloadLength is stored, indicating
- /// the length of the payload at the current term position.</li>
- /// <li>For example, the TermPositions for a term which occurs as the fourth term in
- /// one document, and as the fifth and ninth term in a subsequent document, would
- /// be the following sequence of VInts (payloads disabled):
- /// <p>4, 5, 4</p></li>
- /// <li>PayloadData is metadata associated with the current term position. If
- /// PayloadLength is stored at the current position, then it indicates the length
- /// of this payload. If PayloadLength is not stored, then this payload has the same
- /// length as the payload at the previous position.</li>
- /// <li>OffsetDelta/2 is the difference between this position's startOffset from the
- /// previous occurrence (or zero, if this is the first occurrence in this document).
- /// If OffsetDelta is odd, then the length (endOffset-startOffset) differs from the
- /// previous occurrence and an OffsetLength follows. Offset data is only written for
- /// <seealso cref="IndexOptions#DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS"/>.</li>
- /// </ul>
- /// </dd>
- /// </dl>
- ///
- /// <a name="Payloads" id="Payloads"></a>
- /// <dl>
- /// <dd>
- /// <b>Payloads and Offsets</b>
- /// <p>The .pay file will store payloads and offsets associated with certain term-document positions.
- /// Some payloads and offsets will be separated out into .pos file, for performance reasons.</p>
- /// <ul>
- /// <li>PayFile(.pay): --> Header, <TermPayloads, TermOffsets?> <sup>TermCount</sup>, Footer</li>
- /// <li>Header --> <seealso cref="CodecUtil#writeHeader CodecHeader"/></li>
- /// <li>TermPayloads --> <PackedPayLengthBlock, SumPayLength, PayData> <sup>PackedPayBlockNum</sup>
- /// <li>TermOffsets --> <PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock> <sup>PackedPayBlockNum</sup>
- /// <li>PackedPayLengthBlock, PackedOffsetStartDeltaBlock, PackedOffsetLengthBlock --> <seealso cref="PackedInts PackedInts"/></li>
- /// <li>SumPayLength --> <seealso cref="DataOutput#writeVInt VInt"/></li>
- /// <li>PayData --> <seealso cref="DataOutput#writeByte byte"/><sup>SumPayLength</sup></li>
- /// <li>Footer --> <seealso cref="CodecUtil#writeFooter CodecFooter"/></li>
- /// </ul>
- /// <p>Notes:</p>
- /// <ul>
- /// <li>The order of TermPayloads/TermOffsets will be the same as TermPositions, note that part of
- /// payload/offsets are stored in .pos.</li>
- /// <li>The procedure how PackedPayLengthBlock and PackedOffsetLengthBlock are generated is the
- /// same as PackedFreqBlock in chapter <a href="#Frequencies">Frequencies and Skip Data</a>.
- /// While PackedStartDeltaBlock follows a same procedure as PackedDocDeltaBlock.</li>
- /// <li>PackedPayBlockNum is always equal to PackedPosBlockNum, for the same term. It is also synonym
- /// for PackedOffsetBlockNum.</li>
- /// <li>SumPayLength is the total length of payloads written within one block, should be the sum
- /// of PayLengths in one packed block.</li>
- /// <li>PayLength in PackedPayLengthBlock is the length of each payload associated with the current
- /// position.</li>
- /// </ul>
- /// </dd>
- /// </dl>
- /// </p>
- ///
- /// @lucene.experimental
- /// </summary>
- [PostingsFormatName("Lucene41")] // LUCENENET specific - using PostingsFormatName attribute to ensure the default name passed from subclasses is the same as this class name
- public sealed class Lucene41PostingsFormat : PostingsFormat
- {
- /// <summary>
- /// Filename extension for document number, frequencies, and skip data.
- /// See chapter: <a href="#Frequencies">Frequencies and Skip Data</a>
- /// </summary>
- public const string DOC_EXTENSION = "doc";
-
- /// <summary>
- /// Filename extension for positions.
- /// See chapter: <a href="#Positions">Positions</a>
- /// </summary>
- public const string POS_EXTENSION = "pos";
-
- /// <summary>
- /// Filename extension for payloads and offsets.
- /// See chapter: <a href="#Payloads">Payloads and Offsets</a>
- /// </summary>
- public const string PAY_EXTENSION = "pay";
-
- private readonly int minTermBlockSize;
- private readonly int maxTermBlockSize;
-
- /// <summary>
- /// Fixed packed block size, number of integers encoded in
- /// a single packed block.
- /// </summary>
- // NOTE: must be multiple of 64 because of PackedInts long-aligned encoding/decoding
- public static int BLOCK_SIZE = 128;
-
- /// <summary>
- /// Creates {@code Lucene41PostingsFormat} with default
- /// settings.
- /// </summary>
- public Lucene41PostingsFormat()
- : this(BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE)
- {
- }
-
- /// <summary>
- /// Creates {@code Lucene41PostingsFormat} with custom
- /// values for {@code minBlockSize} and {@code
- /// maxBlockSize} passed to block terms dictionary. </summary>
- /// <seealso cref= BlockTreeTermsWriter#BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int) </seealso>
- public Lucene41PostingsFormat(int minTermBlockSize, int maxTermBlockSize)
- : base()
- {
- this.minTermBlockSize = minTermBlockSize;
- Debug.Assert(minTermBlockSize > 1);
- this.maxTermBlockSize = maxTermBlockSize;
- Debug.Assert(minTermBlockSize <= maxTermBlockSize);
- }
-
- public override string ToString()
- {
- return Name + "(blocksize=" + BLOCK_SIZE + ")";
- }
-
- public override FieldsConsumer FieldsConsumer(SegmentWriteState state)
- {
- PostingsWriterBase postingsWriter = new Lucene41PostingsWriter(state);
-
- bool success = false;
- try
- {
- FieldsConsumer ret = new BlockTreeTermsWriter(state, postingsWriter, minTermBlockSize, maxTermBlockSize);
- success = true;
- return ret;
- }
- finally
- {
- if (!success)
- {
- IOUtils.CloseWhileHandlingException(postingsWriter);
- }
- }
- }
-
- public override FieldsProducer FieldsProducer(SegmentReadState state)
- {
- PostingsReaderBase postingsReader = new Lucene41PostingsReader(state.Directory, state.FieldInfos, state.SegmentInfo, state.Context, state.SegmentSuffix);
- bool success = false;
- try
- {
- FieldsProducer ret = new BlockTreeTermsReader(state.Directory, state.FieldInfos, state.SegmentInfo, postingsReader, state.Context, state.SegmentSuffix, state.TermsIndexDivisor);
- success = true;
- return ret;
- }
- finally
- {
- if (!success)
- {
- IOUtils.CloseWhileHandlingException(postingsReader);
- }
- }
- }
- }
-}
\ No newline at end of file