You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by cc...@apache.org on 2013/04/03 19:40:07 UTC
[24/51] [partial] Mass convert mixed tabs to spaces
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Token.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Token.cs b/src/core/Analysis/Token.cs
index 3357f34..ec5e29b 100644
--- a/src/core/Analysis/Token.cs
+++ b/src/core/Analysis/Token.cs
@@ -26,780 +26,780 @@ using Attribute = Lucene.Net.Util.Attribute;
namespace Lucene.Net.Analysis
{
-
- /// <summary>A Token is an occurrence of a term from the text of a field. It consists of
- /// a term's text, the start and end offset of the term in the text of the field,
- /// and a type string.
- /// <p/>
- /// The start and end offsets permit applications to re-associate a token with
- /// its source text, e.g., to display highlighted query terms in a document
- /// browser, or to show matching text fragments in a <abbr
- /// title="KeyWord In Context">KWIC</abbr> display, etc.
- /// <p/>
- /// The type is a string, assigned by a lexical analyzer
- /// (a.k.a. tokenizer), naming the lexical or syntactic class that the token
- /// belongs to. For example an end of sentence marker token might be implemented
- /// with type "eos". The default token type is "word".
- /// <p/>
- /// A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
- /// length byte array. Use <see cref="TermPositions.PayloadLength" /> and
- /// <see cref="TermPositions.GetPayload(byte[], int)" /> to retrieve the payloads from the index.
- /// </summary>
- /// <summary><br/><br/>
- /// </summary>
- /// <summary><p/><b>NOTE:</b> As of 2.9, Token implements all <see cref="IAttribute" /> interfaces
- /// that are part of core Lucene and can be found in the <see cref="Lucene.Net.Analysis.Tokenattributes"/> namespace.
- /// Even though it is not necessary to use Token anymore, with the new TokenStream API it can
- /// be used as convenience class that implements all <see cref="IAttribute" />s, which is especially useful
- /// to easily switch from the old to the new TokenStream API.
- /// <br/><br/>
- /// <p/>Tokenizers and TokenFilters should try to re-use a Token instance when
- /// possible for best performance, by implementing the
- /// <see cref="TokenStream.IncrementToken()" /> API.
- /// Failing that, to create a new Token you should first use
- /// one of the constructors that starts with null text. To load
- /// the token from a char[] use <see cref="SetTermBuffer(char[], int, int)" />.
- /// To load from a String use <see cref="SetTermBuffer(String)" /> or <see cref="SetTermBuffer(String, int, int)" />.
- /// Alternatively you can get the Token's termBuffer by calling either <see cref="TermBuffer()" />,
- /// if you know that your text is shorter than the capacity of the termBuffer
- /// or <see cref="ResizeTermBuffer(int)" />, if there is any possibility
- /// that you may need to grow the buffer. Fill in the characters of your term into this
+
+ /// <summary>A Token is an occurrence of a term from the text of a field. It consists of
+ /// a term's text, the start and end offset of the term in the text of the field,
+ /// and a type string.
+ /// <p/>
+ /// The start and end offsets permit applications to re-associate a token with
+ /// its source text, e.g., to display highlighted query terms in a document
+ /// browser, or to show matching text fragments in a <abbr
+ /// title="KeyWord In Context">KWIC</abbr> display, etc.
+ /// <p/>
+ /// The type is a string, assigned by a lexical analyzer
+ /// (a.k.a. tokenizer), naming the lexical or syntactic class that the token
+ /// belongs to. For example an end of sentence marker token might be implemented
+ /// with type "eos". The default token type is "word".
+ /// <p/>
+ /// A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
+ /// length byte array. Use <see cref="TermPositions.PayloadLength" /> and
+ /// <see cref="TermPositions.GetPayload(byte[], int)" /> to retrieve the payloads from the index.
+ /// </summary>
+ /// <summary><br/><br/>
+ /// </summary>
+ /// <summary><p/><b>NOTE:</b> As of 2.9, Token implements all <see cref="IAttribute" /> interfaces
+ /// that are part of core Lucene and can be found in the <see cref="Lucene.Net.Analysis.Tokenattributes"/> namespace.
+ /// Even though it is not necessary to use Token anymore, with the new TokenStream API it can
+ /// be used as convenience class that implements all <see cref="IAttribute" />s, which is especially useful
+ /// to easily switch from the old to the new TokenStream API.
+ /// <br/><br/>
+ /// <p/>Tokenizers and TokenFilters should try to re-use a Token instance when
+ /// possible for best performance, by implementing the
+ /// <see cref="TokenStream.IncrementToken()" /> API.
+ /// Failing that, to create a new Token you should first use
+ /// one of the constructors that starts with null text. To load
+ /// the token from a char[] use <see cref="SetTermBuffer(char[], int, int)" />.
+ /// To load from a String use <see cref="SetTermBuffer(String)" /> or <see cref="SetTermBuffer(String, int, int)" />.
+ /// Alternatively you can get the Token's termBuffer by calling either <see cref="TermBuffer()" />,
+ /// if you know that your text is shorter than the capacity of the termBuffer
+ /// or <see cref="ResizeTermBuffer(int)" />, if there is any possibility
+ /// that you may need to grow the buffer. Fill in the characters of your term into this
/// buffer, with <see cref="string.ToCharArray(int, int)" /> if loading from a string,
- /// or with <see cref="Array.Copy(Array, long, Array, long, long)" />, and finally call <see cref="SetTermLength(int)" /> to
- /// set the length of the term text. See <a target="_top"
- /// href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
- /// for details.<p/>
- /// <p/>Typical Token reuse patterns:
- /// <list type="bullet">
- /// <item> Copying text from a string (type is reset to <see cref="DEFAULT_TYPE" /> if not
- /// specified):<br/>
- /// <code>
- /// return reusableToken.reinit(string, startOffset, endOffset[, type]);
- /// </code>
- /// </item>
- /// <item> Copying some text from a string (type is reset to <see cref="DEFAULT_TYPE" />
- /// if not specified):<br/>
+ /// or with <see cref="Array.Copy(Array, long, Array, long, long)" />, and finally call <see cref="SetTermLength(int)" /> to
+ /// set the length of the term text. See <a target="_top"
+ /// href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
+ /// for details.<p/>
+ /// <p/>Typical Token reuse patterns:
+ /// <list type="bullet">
+ /// <item> Copying text from a string (type is reset to <see cref="DEFAULT_TYPE" /> if not
+ /// specified):<br/>
/// <code>
- /// return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
+ /// return reusableToken.reinit(string, startOffset, endOffset[, type]);
/// </code>
- /// </item>
- /// <item> Copying text from char[] buffer (type is reset to <see cref="DEFAULT_TYPE" />
- /// if not specified):<br/>
+ /// </item>
+ /// <item> Copying some text from a string (type is reset to <see cref="DEFAULT_TYPE" />
+ /// if not specified):<br/>
/// <code>
- /// return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
+ /// return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
/// </code>
- /// </item>
- /// <item> Copying some text from a char[] buffer (type is reset to
- /// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
+ /// </item>
+ /// <item> Copying text from char[] buffer (type is reset to <see cref="DEFAULT_TYPE" />
+ /// if not specified):<br/>
/// <code>
- /// return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
+ /// return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
/// </code>
- /// </item>
- /// <item> Copying from one one Token to another (type is reset to
- /// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
+ /// </item>
+ /// <item> Copying some text from a char[] buffer (type is reset to
+ /// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
/// <code>
- /// return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]);
+ /// return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
/// </code>
- /// </item>
- /// </list>
- /// A few things to note:
- /// <list type="bullet">
- /// <item>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</item>
- /// <item>Because <c>TokenStreams</c> can be chained, one cannot assume that the <c>Token's</c> current type is correct.</item>
- /// <item>The startOffset and endOffset represent the start and offset in the
- /// source text, so be careful in adjusting them.</item>
- /// <item>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</item>
- /// </list>
- /// <p/>
- /// </summary>
- /// <seealso cref="Lucene.Net.Index.Payload">
- /// </seealso>
- [Serializable]
- public class Token : Attribute, ITermAttribute, ITypeAttribute, IPositionIncrementAttribute, IFlagsAttribute, IOffsetAttribute, IPayloadAttribute
- {
- public const String DEFAULT_TYPE = "word";
+ /// </item>
+ /// <item> Copying from one one Token to another (type is reset to
+ /// <see cref="DEFAULT_TYPE" /> if not specified):<br/>
+ /// <code>
+ /// return reusableToken.reinit(source.termBuffer(), 0, source.termLength(), source.startOffset(), source.endOffset()[, source.type()]);
+ /// </code>
+ /// </item>
+ /// </list>
+ /// A few things to note:
+ /// <list type="bullet">
+ /// <item>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</item>
+ /// <item>Because <c>TokenStreams</c> can be chained, one cannot assume that the <c>Token's</c> current type is correct.</item>
+ /// <item>The startOffset and endOffset represent the start and offset in the
+ /// source text, so be careful in adjusting them.</item>
+ /// <item>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</item>
+ /// </list>
+ /// <p/>
+ /// </summary>
+ /// <seealso cref="Lucene.Net.Index.Payload">
+ /// </seealso>
+ [Serializable]
+ public class Token : Attribute, ITermAttribute, ITypeAttribute, IPositionIncrementAttribute, IFlagsAttribute, IOffsetAttribute, IPayloadAttribute
+ {
+ public const String DEFAULT_TYPE = "word";
- private const int MIN_BUFFER_SIZE = 10;
+ private const int MIN_BUFFER_SIZE = 10;
- private char[] termBuffer;
- private int termLength;
- private int startOffset, endOffset;
- private string type = DEFAULT_TYPE;
- private int flags;
- private Payload payload;
- private int positionIncrement = 1;
-
- /// <summary>Constructs a Token will null text. </summary>
- public Token()
- {
- }
-
- /// <summary>Constructs a Token with null text and start & end
- /// offsets.
- /// </summary>
- /// <param name="start">start offset in the source text</param>
- /// <param name="end">end offset in the source text</param>
- public Token(int start, int end)
- {
- startOffset = start;
- endOffset = end;
- }
-
- /// <summary>Constructs a Token with null text and start & end
- /// offsets plus the Token type.
- /// </summary>
- /// <param name="start">start offset in the source text</param>
- /// <param name="end">end offset in the source text</param>
- /// <param name="typ">the lexical type of this Token</param>
- public Token(int start, int end, String typ)
- {
- startOffset = start;
- endOffset = end;
- type = typ;
- }
-
- /// <summary> Constructs a Token with null text and start & end
- /// offsets plus flags. NOTE: flags is EXPERIMENTAL.
- /// </summary>
- /// <param name="start">start offset in the source text</param>
- /// <param name="end">end offset in the source text</param>
- /// <param name="flags">The bits to set for this token</param>
- public Token(int start, int end, int flags)
- {
- startOffset = start;
- endOffset = end;
- this.flags = flags;
- }
-
- /// <summary>Constructs a Token with the given term text, and start
- /// & end offsets. The type defaults to "word."
- /// <b>NOTE:</b> for better indexing speed you should
- /// instead use the char[] termBuffer methods to set the
- /// term text.
- /// </summary>
- /// <param name="text">term text</param>
- /// <param name="start">start offset</param>
- /// <param name="end">end offset</param>
- public Token(String text, int start, int end)
- {
- SetTermBuffer(text);
- startOffset = start;
- endOffset = end;
- }
-
- /// <summary>Constructs a Token with the given text, start and end
- /// offsets, & type. <b>NOTE:</b> for better indexing
- /// speed you should instead use the char[] termBuffer
- /// methods to set the term text.
- /// </summary>
- /// <param name="text">term text</param>
- /// <param name="start">start offset</param>
- /// <param name="end">end offset</param>
- /// <param name="typ">token type</param>
- public Token(System.String text, int start, int end, System.String typ)
- {
- SetTermBuffer(text);
- startOffset = start;
- endOffset = end;
- type = typ;
- }
-
- /// <summary> Constructs a Token with the given text, start and end
- /// offsets, & type. <b>NOTE:</b> for better indexing
- /// speed you should instead use the char[] termBuffer
- /// methods to set the term text.
- /// </summary>
- /// <param name="text"></param>
- /// <param name="start"></param>
- /// <param name="end"></param>
- /// <param name="flags">token type bits</param>
- public Token(System.String text, int start, int end, int flags)
- {
- SetTermBuffer(text);
- startOffset = start;
- endOffset = end;
- this.flags = flags;
- }
-
- /// <summary> Constructs a Token with the given term buffer (offset
- /// & length), start and end
- /// offsets
- /// </summary>
- /// <param name="startTermBuffer"></param>
- /// <param name="termBufferOffset"></param>
- /// <param name="termBufferLength"></param>
- /// <param name="start"></param>
- /// <param name="end"></param>
- public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
- {
- SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
- startOffset = start;
- endOffset = end;
- }
+ private char[] termBuffer;
+ private int termLength;
+ private int startOffset, endOffset;
+ private string type = DEFAULT_TYPE;
+ private int flags;
+ private Payload payload;
+ private int positionIncrement = 1;
+
+ /// <summary>Constructs a Token will null text. </summary>
+ public Token()
+ {
+ }
+
+ /// <summary>Constructs a Token with null text and start & end
+ /// offsets.
+ /// </summary>
+ /// <param name="start">start offset in the source text</param>
+ /// <param name="end">end offset in the source text</param>
+ public Token(int start, int end)
+ {
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /// <summary>Constructs a Token with null text and start & end
+ /// offsets plus the Token type.
+ /// </summary>
+ /// <param name="start">start offset in the source text</param>
+ /// <param name="end">end offset in the source text</param>
+ /// <param name="typ">the lexical type of this Token</param>
+ public Token(int start, int end, String typ)
+ {
+ startOffset = start;
+ endOffset = end;
+ type = typ;
+ }
+
+ /// <summary> Constructs a Token with null text and start & end
+ /// offsets plus flags. NOTE: flags is EXPERIMENTAL.
+ /// </summary>
+ /// <param name="start">start offset in the source text</param>
+ /// <param name="end">end offset in the source text</param>
+ /// <param name="flags">The bits to set for this token</param>
+ public Token(int start, int end, int flags)
+ {
+ startOffset = start;
+ endOffset = end;
+ this.flags = flags;
+ }
+
+ /// <summary>Constructs a Token with the given term text, and start
+ /// & end offsets. The type defaults to "word."
+ /// <b>NOTE:</b> for better indexing speed you should
+ /// instead use the char[] termBuffer methods to set the
+ /// term text.
+ /// </summary>
+ /// <param name="text">term text</param>
+ /// <param name="start">start offset</param>
+ /// <param name="end">end offset</param>
+ public Token(String text, int start, int end)
+ {
+ SetTermBuffer(text);
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /// <summary>Constructs a Token with the given text, start and end
+ /// offsets, & type. <b>NOTE:</b> for better indexing
+ /// speed you should instead use the char[] termBuffer
+ /// methods to set the term text.
+ /// </summary>
+ /// <param name="text">term text</param>
+ /// <param name="start">start offset</param>
+ /// <param name="end">end offset</param>
+ /// <param name="typ">token type</param>
+ public Token(System.String text, int start, int end, System.String typ)
+ {
+ SetTermBuffer(text);
+ startOffset = start;
+ endOffset = end;
+ type = typ;
+ }
+
+ /// <summary> Constructs a Token with the given text, start and end
+ /// offsets, & type. <b>NOTE:</b> for better indexing
+ /// speed you should instead use the char[] termBuffer
+ /// methods to set the term text.
+ /// </summary>
+ /// <param name="text"></param>
+ /// <param name="start"></param>
+ /// <param name="end"></param>
+ /// <param name="flags">token type bits</param>
+ public Token(System.String text, int start, int end, int flags)
+ {
+ SetTermBuffer(text);
+ startOffset = start;
+ endOffset = end;
+ this.flags = flags;
+ }
+
+ /// <summary> Constructs a Token with the given term buffer (offset
+ /// & length), start and end
+ /// offsets
+ /// </summary>
+ /// <param name="startTermBuffer"></param>
+ /// <param name="termBufferOffset"></param>
+ /// <param name="termBufferLength"></param>
+ /// <param name="start"></param>
+ /// <param name="end"></param>
+ public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end)
+ {
+ SetTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
+ startOffset = start;
+ endOffset = end;
+ }
- /// <summary>Set the position increment. This determines the position of this token
- /// relative to the previous Token in a <see cref="TokenStream" />, used in phrase
- /// searching.
- ///
- /// <p/>The default value is one.
- ///
- /// <p/>Some common uses for this are:<list>
- ///
- /// <item>Set it to zero to put multiple terms in the same position. This is
- /// useful if, e.g., a word has multiple stems. Searches for phrases
- /// including either stem will match. In this case, all but the first stem's
- /// increment should be set to zero: the increment of the first instance
- /// should be one. Repeating a token with an increment of zero can also be
- /// used to boost the scores of matches on that token.</item>
- ///
- /// <item>Set it to values greater than one to inhibit exact phrase matches.
- /// If, for example, one does not want phrases to match across removed stop
- /// words, then one could build a stop word filter that removes stop words and
- /// also sets the increment to the number of stop words removed before each
- /// non-stop word. Then exact phrase queries will only match when the terms
- /// occur with no intervening stop words.</item>
- ///
- /// </list>
- /// </summary>
- /// <value> the distance from the prior term </value>
- /// <seealso cref="Lucene.Net.Index.TermPositions">
- /// </seealso>
- public virtual int PositionIncrement
- {
- set
- {
- if (value < 0)
- throw new System.ArgumentException("Increment must be zero or greater: " + value);
- this.positionIncrement = value;
- }
- get { return positionIncrement; }
- }
+ /// <summary>Set the position increment. This determines the position of this token
+ /// relative to the previous Token in a <see cref="TokenStream" />, used in phrase
+ /// searching.
+ ///
+ /// <p/>The default value is one.
+ ///
+ /// <p/>Some common uses for this are:<list>
+ ///
+ /// <item>Set it to zero to put multiple terms in the same position. This is
+ /// useful if, e.g., a word has multiple stems. Searches for phrases
+ /// including either stem will match. In this case, all but the first stem's
+ /// increment should be set to zero: the increment of the first instance
+ /// should be one. Repeating a token with an increment of zero can also be
+ /// used to boost the scores of matches on that token.</item>
+ ///
+ /// <item>Set it to values greater than one to inhibit exact phrase matches.
+ /// If, for example, one does not want phrases to match across removed stop
+ /// words, then one could build a stop word filter that removes stop words and
+ /// also sets the increment to the number of stop words removed before each
+ /// non-stop word. Then exact phrase queries will only match when the terms
+ /// occur with no intervening stop words.</item>
+ ///
+ /// </list>
+ /// </summary>
+ /// <value> the distance from the prior term </value>
+ /// <seealso cref="Lucene.Net.Index.TermPositions">
+ /// </seealso>
+ public virtual int PositionIncrement
+ {
+ set
+ {
+ if (value < 0)
+ throw new System.ArgumentException("Increment must be zero or greater: " + value);
+ this.positionIncrement = value;
+ }
+ get { return positionIncrement; }
+ }
- /// <summary>Returns the Token's term text.
- ///
- /// This method has a performance penalty
- /// because the text is stored internally in a char[]. If
- /// possible, use <see cref="TermBuffer()" /> and <see cref="TermLength()"/>
- /// directly instead. If you really need a
- /// String, use this method, which is nothing more than
- /// a convenience call to <b>new String(token.termBuffer(), 0, token.termLength())</b>
- /// </summary>
- public string Term
- {
- get
- {
- InitTermBuffer();
- return new System.String(termBuffer, 0, termLength);
- }
- }
+ /// <summary>Returns the Token's term text.
+ ///
+ /// This method has a performance penalty
+ /// because the text is stored internally in a char[]. If
+ /// possible, use <see cref="TermBuffer()" /> and <see cref="TermLength()"/>
+ /// directly instead. If you really need a
+ /// String, use this method, which is nothing more than
+ /// a convenience call to <b>new String(token.termBuffer(), 0, token.termLength())</b>
+ /// </summary>
+ public string Term
+ {
+ get
+ {
+ InitTermBuffer();
+ return new System.String(termBuffer, 0, termLength);
+ }
+ }
- /// <summary>Copies the contents of buffer, starting at offset for
- /// length characters, into the termBuffer array.
- /// </summary>
- /// <param name="buffer">the buffer to copy</param>
- /// <param name="offset">the index in the buffer of the first character to copy</param>
- /// <param name="length">the number of characters to copy</param>
- public void SetTermBuffer(char[] buffer, int offset, int length)
- {
- GrowTermBuffer(length);
- Array.Copy(buffer, offset, termBuffer, 0, length);
- termLength = length;
- }
-
- /// <summary>Copies the contents of buffer into the termBuffer array.</summary>
- /// <param name="buffer">the buffer to copy
- /// </param>
- public void SetTermBuffer(System.String buffer)
- {
- int length = buffer.Length;
- GrowTermBuffer(length);
- TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
- termLength = length;
- }
-
- /// <summary>Copies the contents of buffer, starting at offset and continuing
- /// for length characters, into the termBuffer array.
- /// </summary>
- /// <param name="buffer">the buffer to copy
- /// </param>
- /// <param name="offset">the index in the buffer of the first character to copy
- /// </param>
- /// <param name="length">the number of characters to copy
- /// </param>
- public void SetTermBuffer(System.String buffer, int offset, int length)
- {
- System.Diagnostics.Debug.Assert(offset <= buffer.Length);
- System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
- GrowTermBuffer(length);
- TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
- termLength = length;
- }
-
- /// <summary>Returns the internal termBuffer character array which
- /// you can then directly alter. If the array is too
- /// small for your token, use <see cref="ResizeTermBuffer(int)" />
- /// to increase it. After
- /// altering the buffer be sure to call <see cref="SetTermLength" />
- /// to record the number of valid
- /// characters that were placed into the termBuffer.
- /// </summary>
- public char[] TermBuffer()
- {
- InitTermBuffer();
- return termBuffer;
- }
-
- /// <summary>Grows the termBuffer to at least size newSize, preserving the
- /// existing content. Note: If the next operation is to change
- /// the contents of the term buffer use
- /// <see cref="SetTermBuffer(char[], int, int)" />,
- /// <see cref="SetTermBuffer(String)" />, or
- /// <see cref="SetTermBuffer(String, int, int)" />
- /// to optimally combine the resize with the setting of the termBuffer.
- /// </summary>
- /// <param name="newSize">minimum size of the new termBuffer
- /// </param>
- /// <returns> newly created termBuffer with length >= newSize
- /// </returns>
- public virtual char[] ResizeTermBuffer(int newSize)
- {
- if (termBuffer == null)
- {
+ /// <summary>Copies the contents of buffer, starting at offset for
+ /// length characters, into the termBuffer array.
+ /// </summary>
+ /// <param name="buffer">the buffer to copy</param>
+ /// <param name="offset">the index in the buffer of the first character to copy</param>
+ /// <param name="length">the number of characters to copy</param>
+ public void SetTermBuffer(char[] buffer, int offset, int length)
+ {
+ GrowTermBuffer(length);
+ Array.Copy(buffer, offset, termBuffer, 0, length);
+ termLength = length;
+ }
+
+ /// <summary>Copies the contents of buffer into the termBuffer array.</summary>
+ /// <param name="buffer">the buffer to copy
+ /// </param>
+ public void SetTermBuffer(System.String buffer)
+ {
+ int length = buffer.Length;
+ GrowTermBuffer(length);
+ TextSupport.GetCharsFromString(buffer, 0, length, termBuffer, 0);
+ termLength = length;
+ }
+
+ /// <summary>Copies the contents of buffer, starting at offset and continuing
+ /// for length characters, into the termBuffer array.
+ /// </summary>
+ /// <param name="buffer">the buffer to copy
+ /// </param>
+ /// <param name="offset">the index in the buffer of the first character to copy
+ /// </param>
+ /// <param name="length">the number of characters to copy
+ /// </param>
+ public void SetTermBuffer(System.String buffer, int offset, int length)
+ {
+ System.Diagnostics.Debug.Assert(offset <= buffer.Length);
+ System.Diagnostics.Debug.Assert(offset + length <= buffer.Length);
+ GrowTermBuffer(length);
+ TextSupport.GetCharsFromString(buffer, offset, offset + length, termBuffer, 0);
+ termLength = length;
+ }
+
+ /// <summary>Returns the internal termBuffer character array which
+ /// you can then directly alter. If the array is too
+ /// small for your token, use <see cref="ResizeTermBuffer(int)" />
+ /// to increase it. After
+ /// altering the buffer be sure to call <see cref="SetTermLength" />
+ /// to record the number of valid
+ /// characters that were placed into the termBuffer.
+ /// </summary>
+ public char[] TermBuffer()
+ {
+ InitTermBuffer();
+ return termBuffer;
+ }
+
+ /// <summary>Grows the termBuffer to at least size newSize, preserving the
+ /// existing content. Note: If the next operation is to change
+ /// the contents of the term buffer use
+ /// <see cref="SetTermBuffer(char[], int, int)" />,
+ /// <see cref="SetTermBuffer(String)" />, or
+ /// <see cref="SetTermBuffer(String, int, int)" />
+ /// to optimally combine the resize with the setting of the termBuffer.
+ /// </summary>
+ /// <param name="newSize">minimum size of the new termBuffer
+ /// </param>
+ /// <returns> newly created termBuffer with length >= newSize
+ /// </returns>
+ public virtual char[] ResizeTermBuffer(int newSize)
+ {
+ if (termBuffer == null)
+ {
termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE ? MIN_BUFFER_SIZE : newSize)];
- }
- else
- {
- if (termBuffer.Length < newSize)
- {
- // Not big enough; create a new array with slight
- // over allocation and preserve content
- var newCharBuffer = new char[ArrayUtil.GetNextSize(newSize)];
- Array.Copy(termBuffer, 0, newCharBuffer, 0, termBuffer.Length);
- termBuffer = newCharBuffer;
- }
- }
- return termBuffer;
- }
-
- /// <summary>Allocates a buffer char[] of at least newSize, without preserving the existing content.
- /// its always used in places that set the content
- /// </summary>
- /// <param name="newSize">minimum size of the buffer
- /// </param>
- private void GrowTermBuffer(int newSize)
- {
- if (termBuffer == null)
- {
- // The buffer is always at least MIN_BUFFER_SIZE
- termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize)];
- }
- else
- {
- if (termBuffer.Length < newSize)
- {
- // Not big enough; create a new array with slight
- // over allocation:
- termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
- }
- }
- }
-
- private void InitTermBuffer()
- {
- if (termBuffer == null)
- {
+ }
+ else
+ {
+ if (termBuffer.Length < newSize)
+ {
+ // Not big enough; create a new array with slight
+ // over allocation and preserve content
+ var newCharBuffer = new char[ArrayUtil.GetNextSize(newSize)];
+ Array.Copy(termBuffer, 0, newCharBuffer, 0, termBuffer.Length);
+ termBuffer = newCharBuffer;
+ }
+ }
+ return termBuffer;
+ }
+
+ /// <summary>Allocates a buffer char[] of at least newSize, without preserving the existing content.
+ /// its always used in places that set the content
+ /// </summary>
+ /// <param name="newSize">minimum size of the buffer
+ /// </param>
+ private void GrowTermBuffer(int newSize)
+ {
+ if (termBuffer == null)
+ {
+ // The buffer is always at least MIN_BUFFER_SIZE
+ termBuffer = new char[ArrayUtil.GetNextSize(newSize < MIN_BUFFER_SIZE?MIN_BUFFER_SIZE:newSize)];
+ }
+ else
+ {
+ if (termBuffer.Length < newSize)
+ {
+ // Not big enough; create a new array with slight
+ // over allocation:
+ termBuffer = new char[ArrayUtil.GetNextSize(newSize)];
+ }
+ }
+ }
+
+ private void InitTermBuffer()
+ {
+ if (termBuffer == null)
+ {
termBuffer = new char[ArrayUtil.GetNextSize(MIN_BUFFER_SIZE)];
termLength = 0;
- }
- }
-
- /// <summary>Return number of valid characters (length of the term)
- /// in the termBuffer array.
- /// </summary>
- public int TermLength()
- {
- InitTermBuffer();
- return termLength;
- }
-
- /// <summary>Set number of valid characters (length of the term) in
- /// the termBuffer array. Use this to truncate the termBuffer
- /// or to synchronize with external manipulation of the termBuffer.
- /// Note: to grow the size of the array,
- /// use <see cref="ResizeTermBuffer(int)" /> first.
- /// </summary>
- /// <param name="length">the truncated length
- /// </param>
- public void SetTermLength(int length)
- {
- InitTermBuffer();
- if (length > termBuffer.Length)
- throw new System.ArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.Length + ")");
- termLength = length;
- }
+ }
+ }
+
+ /// <summary>Return number of valid characters (length of the term)
+ /// in the termBuffer array.
+ /// </summary>
+ public int TermLength()
+ {
+ InitTermBuffer();
+ return termLength;
+ }
+
+ /// <summary>Set number of valid characters (length of the term) in
+ /// the termBuffer array. Use this to truncate the termBuffer
+ /// or to synchronize with external manipulation of the termBuffer.
+ /// Note: to grow the size of the array,
+ /// use <see cref="ResizeTermBuffer(int)" /> first.
+ /// </summary>
+ /// <param name="length">the truncated length
+ /// </param>
+ public void SetTermLength(int length)
+ {
+ InitTermBuffer();
+ if (length > termBuffer.Length)
+ throw new System.ArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.Length + ")");
+ termLength = length;
+ }
- /// <summary>Gets or sets this Token's starting offset, the position of the first character
- /// corresponding to this token in the source text.
- /// Note that the difference between endOffset() and startOffset() may not be
- /// equal to <see cref="TermLength"/>, as the term text may have been altered by a
- /// stemmer or some other filter.
- /// </summary>
- public virtual int StartOffset
- {
- get { return startOffset; }
- set { this.startOffset = value; }
- }
+ /// <summary>Gets or sets this Token's starting offset, the position of the first character
+ /// corresponding to this token in the source text.
+ /// Note that the difference between endOffset() and startOffset() may not be
+ /// equal to <see cref="TermLength"/>, as the term text may have been altered by a
+ /// stemmer or some other filter.
+ /// </summary>
+ public virtual int StartOffset
+ {
+ get { return startOffset; }
+ set { this.startOffset = value; }
+ }
- /// <summary>Gets or sets this Token's ending offset, one greater than the position of the
- /// last character corresponding to this token in the source text. The length
- /// of the token in the source text is (endOffset - startOffset).
- /// </summary>
- public virtual int EndOffset
- {
- get { return endOffset; }
- set { this.endOffset = value; }
- }
+ /// <summary>Gets or sets this Token's ending offset, one greater than the position of the
+ /// last character corresponding to this token in the source text. The length
+ /// of the token in the source text is (endOffset - startOffset).
+ /// </summary>
+ public virtual int EndOffset
+ {
+ get { return endOffset; }
+ set { this.endOffset = value; }
+ }
- /// <summary>Set the starting and ending offset.
- /// See StartOffset() and EndOffset()
- /// </summary>
- public virtual void SetOffset(int startOffset, int endOffset)
- {
- this.startOffset = startOffset;
- this.endOffset = endOffset;
- }
+ /// <summary>Set the starting and ending offset.
+ /// See StartOffset() and EndOffset()
+ /// </summary>
+ public virtual void SetOffset(int startOffset, int endOffset)
+ {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
- /// <summary>Returns this Token's lexical type. Defaults to "word". </summary>
- public string Type
- {
- get { return type; }
- set { this.type = value; }
- }
+ /// <summary>Returns this Token's lexical type. Defaults to "word". </summary>
+ public string Type
+ {
+ get { return type; }
+ set { this.type = value; }
+ }
- /// <summary> EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
- /// <p/>
- ///
- /// Get the bitset for any bits that have been set. This is completely distinct from <see cref="Type()" />, although they do share similar purposes.
- /// The flags can be used to encode information about the token for use by other <see cref="TokenFilter"/>s.
- ///
- ///
- /// </summary>
- /// <value> The bits </value>
- public virtual int Flags
- {
- get { return flags; }
- set { flags = value; }
- }
+ /// <summary> EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
+ /// <p/>
+ ///
+ /// Get the bitset for any bits that have been set. This is completely distinct from <see cref="Type()" />, although they do share similar purposes.
+ /// The flags can be used to encode information about the token for use by other <see cref="TokenFilter"/>s.
+ ///
+ ///
+ /// </summary>
+ /// <value> The bits </value>
+ public virtual int Flags
+ {
+ get { return flags; }
+ set { flags = value; }
+ }
- /// <summary> Returns this Token's payload.</summary>
- public virtual Payload Payload
- {
- get { return payload; }
- set { payload = value; }
- }
+ /// <summary> Returns this Token's payload.</summary>
+ public virtual Payload Payload
+ {
+ get { return payload; }
+ set { payload = value; }
+ }
- public override String ToString()
- {
- var sb = new System.Text.StringBuilder();
- sb.Append('(');
- InitTermBuffer();
- if (termBuffer == null)
- sb.Append("null");
- else
- sb.Append(termBuffer, 0, termLength);
- sb.Append(',').Append(startOffset).Append(',').Append(endOffset);
- if (!type.Equals("word"))
- sb.Append(",type=").Append(type);
- if (positionIncrement != 1)
- sb.Append(",posIncr=").Append(positionIncrement);
- sb.Append(')');
- return sb.ToString();
- }
-
- /// <summary>Resets the term text, payload, flags, and positionIncrement,
- /// startOffset, endOffset and token type to default.
- /// </summary>
- public override void Clear()
- {
- payload = null;
- // Leave termBuffer to allow re-use
- termLength = 0;
- positionIncrement = 1;
- flags = 0;
- startOffset = endOffset = 0;
- type = DEFAULT_TYPE;
- }
-
- public override System.Object Clone()
- {
- var t = (Token) base.Clone();
- // Do a deep clone
- if (termBuffer != null)
- {
- t.termBuffer = new char[termBuffer.Length];
- termBuffer.CopyTo(t.termBuffer, 0);
- }
- if (payload != null)
- {
- t.payload = (Payload) payload.Clone();
- }
- return t;
- }
-
- /// <summary>Makes a clone, but replaces the term buffer &
- /// start/end offset in the process. This is more
- /// efficient than doing a full clone (and then calling
- /// setTermBuffer) because it saves a wasted copy of the old
- /// termBuffer.
- /// </summary>
- public virtual Token Clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
- {
- var t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset)
- {positionIncrement = positionIncrement, flags = flags, type = type};
- if (payload != null)
- t.payload = (Payload) payload.Clone();
- return t;
- }
-
- public override bool Equals(Object obj)
- {
- if (obj == this)
- return true;
+ public override String ToString()
+ {
+ var sb = new System.Text.StringBuilder();
+ sb.Append('(');
+ InitTermBuffer();
+ if (termBuffer == null)
+ sb.Append("null");
+ else
+ sb.Append(termBuffer, 0, termLength);
+ sb.Append(',').Append(startOffset).Append(',').Append(endOffset);
+ if (!type.Equals("word"))
+ sb.Append(",type=").Append(type);
+ if (positionIncrement != 1)
+ sb.Append(",posIncr=").Append(positionIncrement);
+ sb.Append(')');
+ return sb.ToString();
+ }
+
+ /// <summary>Resets the term text, payload, flags, and positionIncrement,
+ /// startOffset, endOffset and token type to default.
+ /// </summary>
+ public override void Clear()
+ {
+ payload = null;
+ // Leave termBuffer to allow re-use
+ termLength = 0;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ public override System.Object Clone()
+ {
+ var t = (Token) base.Clone();
+ // Do a deep clone
+ if (termBuffer != null)
+ {
+ t.termBuffer = new char[termBuffer.Length];
+ termBuffer.CopyTo(t.termBuffer, 0);
+ }
+ if (payload != null)
+ {
+ t.payload = (Payload) payload.Clone();
+ }
+ return t;
+ }
+
+ /// <summary>Makes a clone, but replaces the term buffer &
+ /// start/end offset in the process. This is more
+ /// efficient than doing a full clone (and then calling
+ /// setTermBuffer) because it saves a wasted copy of the old
+ /// termBuffer.
+ /// </summary>
+ public virtual Token Clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
+ {
+ var t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset)
+ {positionIncrement = positionIncrement, flags = flags, type = type};
+ if (payload != null)
+ t.payload = (Payload) payload.Clone();
+ return t;
+ }
+
+ public override bool Equals(Object obj)
+ {
+ if (obj == this)
+ return true;
- var other = obj as Token;
- if (other == null)
- return false;
-
- InitTermBuffer();
- other.InitTermBuffer();
+ var other = obj as Token;
+ if (other == null)
+ return false;
+
+ InitTermBuffer();
+ other.InitTermBuffer();
- if (termLength == other.termLength && startOffset == other.startOffset && endOffset == other.endOffset &&
- flags == other.flags && positionIncrement == other.positionIncrement && SubEqual(type, other.type) &&
- SubEqual(payload, other.payload))
- {
- for (int i = 0; i < termLength; i++)
- if (termBuffer[i] != other.termBuffer[i])
- return false;
- return true;
- }
- return false;
- }
-
- private bool SubEqual(System.Object o1, System.Object o2)
- {
- if (o1 == null)
- return o2 == null;
- return o1.Equals(o2);
- }
+ if (termLength == other.termLength && startOffset == other.startOffset && endOffset == other.endOffset &&
+ flags == other.flags && positionIncrement == other.positionIncrement && SubEqual(type, other.type) &&
+ SubEqual(payload, other.payload))
+ {
+ for (int i = 0; i < termLength; i++)
+ if (termBuffer[i] != other.termBuffer[i])
+ return false;
+ return true;
+ }
+ return false;
+ }
+
+ private bool SubEqual(System.Object o1, System.Object o2)
+ {
+ if (o1 == null)
+ return o2 == null;
+ return o1.Equals(o2);
+ }
- public override int GetHashCode()
- {
- InitTermBuffer();
- int code = termLength;
- code = code * 31 + startOffset;
- code = code * 31 + endOffset;
- code = code * 31 + flags;
- code = code * 31 + positionIncrement;
- code = code * 31 + type.GetHashCode();
- code = (payload == null?code:code * 31 + payload.GetHashCode());
- code = code * 31 + ArrayUtil.HashCode(termBuffer, 0, termLength);
- return code;
- }
-
- // like clear() but doesn't clear termBuffer/text
- private void ClearNoTermBuffer()
- {
- payload = null;
- positionIncrement = 1;
- flags = 0;
- startOffset = endOffset = 0;
- type = DEFAULT_TYPE;
- }
-
- /// <summary>Shorthand for calling <see cref="Clear" />,
- /// <see cref="SetTermBuffer(char[], int, int)" />,
- /// <see cref="StartOffset" />,
- /// <see cref="EndOffset" />,
- /// <see cref="Type" />
- /// </summary>
- /// <returns> this Token instance
- /// </returns>
- public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
- {
- ClearNoTermBuffer();
- payload = null;
- positionIncrement = 1;
- SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
- startOffset = newStartOffset;
- endOffset = newEndOffset;
- type = newType;
- return this;
- }
-
- /// <summary>Shorthand for calling <see cref="Clear" />,
- /// <see cref="SetTermBuffer(char[], int, int)" />,
- /// <see cref="StartOffset" />,
- /// <see cref="EndOffset" />
- /// <see cref="Type" /> on Token.DEFAULT_TYPE
- /// </summary>
- /// <returns> this Token instance
- /// </returns>
- public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
- {
- ClearNoTermBuffer();
- SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
- startOffset = newStartOffset;
- endOffset = newEndOffset;
- type = DEFAULT_TYPE;
- return this;
- }
-
- /// <summary>Shorthand for calling <see cref="Clear" />,
- /// <see cref="SetTermBuffer(String)" />,
- /// <see cref="StartOffset" />,
- /// <see cref="EndOffset" />
- /// <see cref="Type" />
- /// </summary>
- /// <returns> this Token instance
- /// </returns>
- public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset, System.String newType)
- {
- ClearNoTermBuffer();
- SetTermBuffer(newTerm);
- startOffset = newStartOffset;
- endOffset = newEndOffset;
- type = newType;
- return this;
- }
-
- /// <summary>Shorthand for calling <see cref="Clear" />,
- /// <see cref="SetTermBuffer(String, int, int)" />,
- /// <see cref="StartOffset" />,
- /// <see cref="EndOffset" />
- /// <see cref="Type" />
- /// </summary>
- /// <returns> this Token instance
- /// </returns>
- public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
- {
- ClearNoTermBuffer();
- SetTermBuffer(newTerm, newTermOffset, newTermLength);
- startOffset = newStartOffset;
- endOffset = newEndOffset;
- type = newType;
- return this;
- }
-
- /// <summary>Shorthand for calling <see cref="Clear" />,
- /// <see cref="SetTermBuffer(String)" />,
- /// <see cref="StartOffset" />,
- /// <see cref="EndOffset" />
- /// <see cref="Type" /> on Token.DEFAULT_TYPE
- /// </summary>
- /// <returns> this Token instance
- /// </returns>
- public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset)
- {
- ClearNoTermBuffer();
- SetTermBuffer(newTerm);
- startOffset = newStartOffset;
- endOffset = newEndOffset;
- type = DEFAULT_TYPE;
- return this;
- }
-
- /// <summary>Shorthand for calling <see cref="Clear" />,
- /// <see cref="SetTermBuffer(String, int, int)" />,
- /// <see cref="StartOffset" />,
- /// <see cref="EndOffset" />
- /// <see cref="Type" /> on Token.DEFAULT_TYPE
- /// </summary>
- /// <returns> this Token instance
- /// </returns>
- public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
- {
- ClearNoTermBuffer();
- SetTermBuffer(newTerm, newTermOffset, newTermLength);
- startOffset = newStartOffset;
- endOffset = newEndOffset;
- type = DEFAULT_TYPE;
- return this;
- }
-
- /// <summary> Copy the prototype token's fields into this one. Note: Payloads are shared.</summary>
- /// <param name="prototype">
- /// </param>
- public virtual void Reinit(Token prototype)
- {
- prototype.InitTermBuffer();
- SetTermBuffer(prototype.termBuffer, 0, prototype.termLength);
- positionIncrement = prototype.positionIncrement;
- flags = prototype.flags;
- startOffset = prototype.startOffset;
- endOffset = prototype.endOffset;
- type = prototype.type;
- payload = prototype.payload;
- }
-
- /// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
- /// <param name="prototype">
- /// </param>
- /// <param name="newTerm">
- /// </param>
- public virtual void Reinit(Token prototype, System.String newTerm)
- {
- SetTermBuffer(newTerm);
- positionIncrement = prototype.positionIncrement;
- flags = prototype.flags;
- startOffset = prototype.startOffset;
- endOffset = prototype.endOffset;
- type = prototype.type;
- payload = prototype.payload;
- }
-
- /// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
- /// <param name="prototype">
- /// </param>
- /// <param name="newTermBuffer">
- /// </param>
- /// <param name="offset">
- /// </param>
- /// <param name="length">
- /// </param>
- public virtual void Reinit(Token prototype, char[] newTermBuffer, int offset, int length)
- {
- SetTermBuffer(newTermBuffer, offset, length);
- positionIncrement = prototype.positionIncrement;
- flags = prototype.flags;
- startOffset = prototype.startOffset;
- endOffset = prototype.endOffset;
- type = prototype.type;
- payload = prototype.payload;
- }
-
- public override void CopyTo(Attribute target)
- {
- if (target is Token)
- {
- var to = (Token) target;
- to.Reinit(this);
- // reinit shares the payload, so clone it:
- if (payload != null)
- {
- to.payload = (Payload) payload.Clone();
- }
- }
- else
- {
- InitTermBuffer();
- ((ITermAttribute) target).SetTermBuffer(termBuffer, 0, termLength);
- ((IOffsetAttribute) target).SetOffset(startOffset, endOffset);
- ((IPositionIncrementAttribute) target).PositionIncrement = positionIncrement;
- ((IPayloadAttribute) target).Payload = (payload == null)?null:(Payload) payload.Clone();
- ((IFlagsAttribute) target).Flags = flags;
- ((ITypeAttribute) target).Type = type;
- }
- }
+ public override int GetHashCode()
+ {
+ InitTermBuffer();
+ int code = termLength;
+ code = code * 31 + startOffset;
+ code = code * 31 + endOffset;
+ code = code * 31 + flags;
+ code = code * 31 + positionIncrement;
+ code = code * 31 + type.GetHashCode();
+ code = (payload == null?code:code * 31 + payload.GetHashCode());
+ code = code * 31 + ArrayUtil.HashCode(termBuffer, 0, termLength);
+ return code;
+ }
+
+ // like clear() but doesn't clear termBuffer/text
+ private void ClearNoTermBuffer()
+ {
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ /// <summary>Shorthand for calling <see cref="Clear" />,
+ /// <see cref="SetTermBuffer(char[], int, int)" />,
+ /// <see cref="StartOffset" />,
+ /// <see cref="EndOffset" />,
+ /// <see cref="Type" />
+ /// </summary>
+ /// <returns> this Token instance
+ /// </returns>
+ public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
+ {
+ ClearNoTermBuffer();
+ payload = null;
+ positionIncrement = 1;
+ SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /// <summary>Shorthand for calling <see cref="Clear" />,
+ /// <see cref="SetTermBuffer(char[], int, int)" />,
+ /// <see cref="StartOffset" />,
+ /// <see cref="EndOffset" />
+ /// <see cref="Type" /> on Token.DEFAULT_TYPE
+ /// </summary>
+ /// <returns> this Token instance
+ /// </returns>
+ public virtual Token Reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
+ {
+ ClearNoTermBuffer();
+ SetTermBuffer(newTermBuffer, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /// <summary>Shorthand for calling <see cref="Clear" />,
+ /// <see cref="SetTermBuffer(String)" />,
+ /// <see cref="StartOffset" />,
+ /// <see cref="EndOffset" />
+ /// <see cref="Type" />
+ /// </summary>
+ /// <returns> this Token instance
+ /// </returns>
+ public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset, System.String newType)
+ {
+ ClearNoTermBuffer();
+ SetTermBuffer(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /// <summary>Shorthand for calling <see cref="Clear" />,
+ /// <see cref="SetTermBuffer(String, int, int)" />,
+ /// <see cref="StartOffset" />,
+ /// <see cref="EndOffset" />
+ /// <see cref="Type" />
+ /// </summary>
+ /// <returns> this Token instance
+ /// </returns>
+ public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, System.String newType)
+ {
+ ClearNoTermBuffer();
+ SetTermBuffer(newTerm, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /// <summary>Shorthand for calling <see cref="Clear" />,
+ /// <see cref="SetTermBuffer(String)" />,
+ /// <see cref="StartOffset" />,
+ /// <see cref="EndOffset" />
+ /// <see cref="Type" /> on Token.DEFAULT_TYPE
+ /// </summary>
+ /// <returns> this Token instance
+ /// </returns>
+ public virtual Token Reinit(System.String newTerm, int newStartOffset, int newEndOffset)
+ {
+ ClearNoTermBuffer();
+ SetTermBuffer(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /// <summary>Shorthand for calling <see cref="Clear" />,
+ /// <see cref="SetTermBuffer(String, int, int)" />,
+ /// <see cref="StartOffset" />,
+ /// <see cref="EndOffset" />
+ /// <see cref="Type" /> on Token.DEFAULT_TYPE
+ /// </summary>
+ /// <returns> this Token instance
+ /// </returns>
+ public virtual Token Reinit(System.String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset)
+ {
+ ClearNoTermBuffer();
+ SetTermBuffer(newTerm, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /// <summary> Copy the prototype token's fields into this one. Note: Payloads are shared.</summary>
+ /// <param name="prototype">
+ /// </param>
+ public virtual void Reinit(Token prototype)
+ {
+ prototype.InitTermBuffer();
+ SetTermBuffer(prototype.termBuffer, 0, prototype.termLength);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
+ /// <param name="prototype">
+ /// </param>
+ /// <param name="newTerm">
+ /// </param>
+ public virtual void Reinit(Token prototype, System.String newTerm)
+ {
+ SetTermBuffer(newTerm);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /// <summary> Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.</summary>
+ /// <param name="prototype">
+ /// </param>
+ /// <param name="newTermBuffer">
+ /// </param>
+ /// <param name="offset">
+ /// </param>
+ /// <param name="length">
+ /// </param>
+ public virtual void Reinit(Token prototype, char[] newTermBuffer, int offset, int length)
+ {
+ SetTermBuffer(newTermBuffer, offset, length);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ public override void CopyTo(Attribute target)
+ {
+ if (target is Token)
+ {
+ var to = (Token) target;
+ to.Reinit(this);
+ // reinit shares the payload, so clone it:
+ if (payload != null)
+ {
+ to.payload = (Payload) payload.Clone();
+ }
+ }
+ else
+ {
+ InitTermBuffer();
+ ((ITermAttribute) target).SetTermBuffer(termBuffer, 0, termLength);
+ ((IOffsetAttribute) target).SetOffset(startOffset, endOffset);
+ ((IPositionIncrementAttribute) target).PositionIncrement = positionIncrement;
+ ((IPayloadAttribute) target).Payload = (payload == null)?null:(Payload) payload.Clone();
+ ((IFlagsAttribute) target).Flags = flags;
+ ((ITypeAttribute) target).Type = type;
+ }
+ }
///<summary>
/// Convenience factory that returns <c>Token</c> as implementation for the basic
@@ -807,8 +807,8 @@ namespace Lucene.Net.Analysis
/// attributes.
/// @since 3.0
/// </summary>
- public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
- new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
+ public static AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
+ new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
/// <summary>
/// <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
@@ -839,8 +839,8 @@ namespace Lucene.Net.Analysis
{
if (this == other) return true;
- var af = other as TokenAttributeFactory;
- return af != null && _delegateFactory.Equals(af._delegateFactory);
+ var af = other as TokenAttributeFactory;
+ return af != null && _delegateFactory.Equals(af._delegateFactory);
}
public override int GetHashCode()
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/TokenFilter.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/TokenFilter.cs b/src/core/Analysis/TokenFilter.cs
index 7483c82..2cd1093 100644
--- a/src/core/Analysis/TokenFilter.cs
+++ b/src/core/Analysis/TokenFilter.cs
@@ -17,35 +17,35 @@
namespace Lucene.Net.Analysis
{
-
- /// <summary> A TokenFilter is a TokenStream whose input is another TokenStream.
- /// <p/>
+
+ /// <summary> A TokenFilter is a TokenStream whose input is another TokenStream.
+ /// <p/>
/// This is an abstract class; subclasses must override <see cref="TokenStream.IncrementToken()" />.
- ///
- /// </summary>
- /// <seealso cref="TokenStream">
- /// </seealso>
- public abstract class TokenFilter:TokenStream
- {
- /// <summary>The source of tokens for this filter. </summary>
- protected internal TokenStream input;
+ ///
+ /// </summary>
+ /// <seealso cref="TokenStream">
+ /// </seealso>
+ public abstract class TokenFilter:TokenStream
+ {
+ /// <summary>The source of tokens for this filter. </summary>
+ protected internal TokenStream input;
- private bool isDisposed;
+ private bool isDisposed;
- /// <summary>Construct a token stream filtering the given input. </summary>
- protected internal TokenFilter(TokenStream input):base(input)
- {
- this.input = input;
- }
-
- /// <summary>Performs end-of-stream operations, if any, and calls then <c>end()</c> on the
- /// input TokenStream.<p/>
- /// <b>NOTE:</b> Be sure to call <c>super.end()</c> first when overriding this method.
- /// </summary>
- public override void End()
- {
- input.End();
- }
+ /// <summary>Construct a token stream filtering the given input. </summary>
+ protected internal TokenFilter(TokenStream input):base(input)
+ {
+ this.input = input;
+ }
+
+ /// <summary>Performs end-of-stream operations, if any, and calls then <c>end()</c> on the
+ /// input TokenStream.<p/>
+ /// <b>NOTE:</b> Be sure to call <c>super.end()</c> first when overriding this method.
+ /// </summary>
+ public override void End()
+ {
+ input.End();
+ }
protected override void Dispose(bool disposing)
{
@@ -62,11 +62,11 @@ namespace Lucene.Net.Analysis
//input = null;
isDisposed = true;
}
-
- /// <summary>Reset the filter as well as the input TokenStream. </summary>
- public override void Reset()
- {
- input.Reset();
- }
- }
+
+ /// <summary>Reset the filter as well as the input TokenStream. </summary>
+ public override void Reset()
+ {
+ input.Reset();
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/TokenStream.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/TokenStream.cs b/src/core/Analysis/TokenStream.cs
index c624696..4a44609 100644
--- a/src/core/Analysis/TokenStream.cs
+++ b/src/core/Analysis/TokenStream.cs
@@ -24,139 +24,139 @@ using AttributeSource = Lucene.Net.Util.AttributeSource;
namespace Lucene.Net.Analysis
{
-
- /// <summary> A <c>TokenStream</c> enumerates the sequence of tokens, either from
- /// <see cref="Field" />s of a <see cref="Document" /> or from query text.
- /// <p/>
- /// This is an abstract class. Concrete subclasses are:
- /// <list type="bullet">
- /// <item><see cref="Tokenizer" />, a <c>TokenStream</c> whose input is a Reader; and</item>
- /// <item><see cref="TokenFilter" />, a <c>TokenStream</c> whose input is another
- /// <c>TokenStream</c>.</item>
- /// </list>
- /// A new <c>TokenStream</c> API has been introduced with Lucene 2.9. This API
- /// has moved from being <see cref="Token" /> based to <see cref="IAttribute" /> based. While
- /// <see cref="Token" /> still exists in 2.9 as a convenience class, the preferred way
- /// to store the information of a <see cref="Token" /> is to use <see cref="Util.Attribute" />s.
- /// <p/>
- /// <c>TokenStream</c> now extends <see cref="AttributeSource" />, which provides
- /// access to all of the token <see cref="IAttribute" />s for the <c>TokenStream</c>.
- /// Note that only one instance per <see cref="Util.Attribute" /> is created and reused
- /// for every token. This approach reduces object creation and allows local
- /// caching of references to the <see cref="Util.Attribute" />s. See
- /// <see cref="IncrementToken()" /> for further details.
- /// <p/>
- /// <b>The workflow of the new <c>TokenStream</c> API is as follows:</b>
- /// <list type="bullet">
- /// <item>Instantiation of <c>TokenStream</c>/<see cref="TokenFilter" />s which add/get
- /// attributes to/from the <see cref="AttributeSource" />.</item>
- /// <item>The consumer calls <see cref="TokenStream.Reset()" />.</item>
- /// <item>The consumer retrieves attributes from the stream and stores local
- /// references to all attributes it wants to access</item>
- /// <item>The consumer calls <see cref="IncrementToken()" /> until it returns false and
- /// consumes the attributes after each call.</item>
- /// <item>The consumer calls <see cref="End()" /> so that any end-of-stream operations
- /// can be performed.</item>
- /// <item>The consumer calls <see cref="Close()" /> to release any resource when finished
- /// using the <c>TokenStream</c></item>
- /// </list>
- /// To make sure that filters and consumers know which attributes are available,
- /// the attributes must be added during instantiation. Filters and consumers are
- /// not required to check for availability of attributes in
- /// <see cref="IncrementToken()" />.
- /// <p/>
- /// You can find some example code for the new API in the analysis package level
- /// Javadoc.
- /// <p/>
- /// Sometimes it is desirable to capture a current state of a <c>TokenStream</c>
- /// , e. g. for buffering purposes (see <see cref="CachingTokenFilter" />,
- /// <see cref="TeeSinkTokenFilter" />). For this usecase
- /// <see cref="AttributeSource.CaptureState" /> and <see cref="AttributeSource.RestoreState" />
- /// can be used.
- /// </summary>
- public abstract class TokenStream : AttributeSource, IDisposable
- {
- /// <summary> A TokenStream using the default attribute factory.</summary>
- protected internal TokenStream()
- { }
-
- /// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
+
+ /// <summary> A <c>TokenStream</c> enumerates the sequence of tokens, either from
+ /// <see cref="Field" />s of a <see cref="Document" /> or from query text.
+ /// <p/>
+ /// This is an abstract class. Concrete subclasses are:
+ /// <list type="bullet">
+ /// <item><see cref="Tokenizer" />, a <c>TokenStream</c> whose input is a Reader; and</item>
+ /// <item><see cref="TokenFilter" />, a <c>TokenStream</c> whose input is another
+ /// <c>TokenStream</c>.</item>
+ /// </list>
+ /// A new <c>TokenStream</c> API has been introduced with Lucene 2.9. This API
+ /// has moved from being <see cref="Token" /> based to <see cref="IAttribute" /> based. While
+ /// <see cref="Token" /> still exists in 2.9 as a convenience class, the preferred way
+ /// to store the information of a <see cref="Token" /> is to use <see cref="Util.Attribute" />s.
+ /// <p/>
+ /// <c>TokenStream</c> now extends <see cref="AttributeSource" />, which provides
+ /// access to all of the token <see cref="IAttribute" />s for the <c>TokenStream</c>.
+ /// Note that only one instance per <see cref="Util.Attribute" /> is created and reused
+ /// for every token. This approach reduces object creation and allows local
+ /// caching of references to the <see cref="Util.Attribute" />s. See
+ /// <see cref="IncrementToken()" /> for further details.
+ /// <p/>
+ /// <b>The workflow of the new <c>TokenStream</c> API is as follows:</b>
+ /// <list type="bullet">
+ /// <item>Instantiation of <c>TokenStream</c>/<see cref="TokenFilter" />s which add/get
+ /// attributes to/from the <see cref="AttributeSource" />.</item>
+ /// <item>The consumer calls <see cref="TokenStream.Reset()" />.</item>
+ /// <item>The consumer retrieves attributes from the stream and stores local
+ /// references to all attributes it wants to access</item>
+ /// <item>The consumer calls <see cref="IncrementToken()" /> until it returns false and
+ /// consumes the attributes after each call.</item>
+ /// <item>The consumer calls <see cref="End()" /> so that any end-of-stream operations
+ /// can be performed.</item>
+ /// <item>The consumer calls <see cref="Close()" /> to release any resource when finished
+ /// using the <c>TokenStream</c></item>
+ /// </list>
+ /// To make sure that filters and consumers know which attributes are available,
+ /// the attributes must be added during instantiation. Filters and consumers are
+ /// not required to check for availability of attributes in
+ /// <see cref="IncrementToken()" />.
+ /// <p/>
+ /// You can find some example code for the new API in the analysis package level
+ /// Javadoc.
+ /// <p/>
+ /// Sometimes it is desirable to capture a current state of a <c>TokenStream</c>
+ /// , e. g. for buffering purposes (see <see cref="CachingTokenFilter" />,
+ /// <see cref="TeeSinkTokenFilter" />). For this usecase
+ /// <see cref="AttributeSource.CaptureState" /> and <see cref="AttributeSource.RestoreState" />
+ /// can be used.
+ /// </summary>
+ public abstract class TokenStream : AttributeSource, IDisposable
+ {
+ /// <summary> A TokenStream using the default attribute factory.</summary>
+ protected internal TokenStream()
+ { }
+
+ /// <summary> A TokenStream that uses the same attributes as the supplied one.</summary>
protected internal TokenStream(AttributeSource input)
: base(input)
- { }
-
- /// <summary> A TokenStream using the supplied AttributeFactory for creating new <see cref="IAttribute" /> instances.</summary>
+ { }
+
+ /// <summary> A TokenStream using the supplied AttributeFactory for creating new <see cref="IAttribute" /> instances.</summary>
protected internal TokenStream(AttributeFactory factory)
: base(factory)
- { }
+ { }
- /// <summary> Consumers (i.e., <see cref="IndexWriter" />) use this method to advance the stream to
- /// the next token. Implementing classes must implement this method and update
- /// the appropriate <see cref="Util.Attribute" />s with the attributes of the next
- /// token.
- ///
- /// The producer must make no assumptions about the attributes after the
- /// method has been returned: the caller may arbitrarily change it. If the
- /// producer needs to preserve the state for subsequent calls, it can use
- /// <see cref="AttributeSource.CaptureState" /> to create a copy of the current attribute state.
- ///
- /// This method is called for every token of a document, so an efficient
- /// implementation is crucial for good performance. To avoid calls to
- /// <see cref="AttributeSource.AddAttribute{T}()" /> and <see cref="AttributeSource.GetAttribute{T}()" />,
- /// references to all <see cref="Util.Attribute" />s that this stream uses should be
- /// retrieved during instantiation.
- ///
- /// To ensure that filters and consumers know which attributes are available,
- /// the attributes must be added during instantiation. Filters and consumers
- /// are not required to check for availability of attributes in
- /// <see cref="IncrementToken()" />.
- ///
- /// </summary>
- /// <returns> false for end of stream; true otherwise</returns>
- public abstract bool IncrementToken();
-
- /// <summary> This method is called by the consumer after the last token has been
- /// consumed, after <see cref="IncrementToken" /> returned <c>false</c>
- /// (using the new <c>TokenStream</c> API). Streams implementing the old API
- /// should upgrade to use this feature.
- /// <p/>
- /// This method can be used to perform any end-of-stream operations, such as
- /// setting the final offset of a stream. The final offset of a stream might
- /// differ from the offset of the last token eg in case one or more whitespaces
- /// followed after the last token, but a <see cref="WhitespaceTokenizer" /> was used.
- ///
- /// </summary>
- /// <throws> IOException </throws>
- public virtual void End()
- {
- // do nothing by default
- }
-
- /// <summary> Resets this stream to the beginning. This is an optional operation, so
- /// subclasses may or may not implement this method. <see cref="Reset()" /> is not needed for
- /// the standard indexing process. However, if the tokens of a
- /// <c>TokenStream</c> are intended to be consumed more than once, it is
- /// necessary to implement <see cref="Reset()" />. Note that if your TokenStream
- /// caches tokens and feeds them back again after a reset, it is imperative
- /// that you clone the tokens when you store them away (on the first pass) as
- /// well as when you return them (on future passes after <see cref="Reset()" />).
- /// </summary>
- public virtual void Reset()
- {
- }
-
- /// <summary>Releases resources associated with this stream. </summary>
- [Obsolete("Use Dispose() instead")]
- public void Close()
- {
+ /// <summary> Consumers (i.e., <see cref="IndexWriter" />) use this method to advance the stream to
+ /// the next token. Implementing classes must implement this method and update
+ /// the appropriate <see cref="Util.Attribute" />s with the attributes of the next
+ /// token.
+ ///
+ /// The producer must make no assumptions about the attributes after the
+ /// method has been returned: the caller may arbitrarily change it. If the
+ /// producer needs to preserve the state for subsequent calls, it can use
+ /// <see cref="AttributeSource.CaptureState" /> to create a copy of the current attribute state.
+ ///
+ /// This method is called for every token of a document, so an efficient
+ /// implementation is crucial for good performance. To avoid calls to
+ /// <see cref="AttributeSource.AddAttribute{T}()" /> and <see cref="AttributeSource.GetAttribute{T}()" />,
+ /// references to all <see cref="Util.Attribute" />s that this stream uses should be
+ /// retrieved during instantiation.
+ ///
+ /// To ensure that filters and consumers know which attributes are available,
+ /// the attributes must be added during instantiation. Filters and consumers
+ /// are not required to check for availability of attributes in
+ /// <see cref="IncrementToken()" />.
+ ///
+ /// </summary>
+ /// <returns> false for end of stream; true otherwise</returns>
+ public abstract bool IncrementToken();
+
+ /// <summary> This method is called by the consumer after the last token has been
+ /// consumed, after <see cref="IncrementToken" /> returned <c>false</c>
+ /// (using the new <c>TokenStream</c> API). Streams implementing the old API
+ /// should upgrade to use this feature.
+ /// <p/>
+ /// This method can be used to perform any end-of-stream operations, such as
+ /// setting the final offset of a stream. The final offset of a stream might
+ /// differ from the offset of the last token eg in case one or more whitespaces
+ /// followed after the last token, but a <see cref="WhitespaceTokenizer" /> was used.
+ ///
+ /// </summary>
+ /// <throws> IOException </throws>
+ public virtual void End()
+ {
+ // do nothing by default
+ }
+
+ /// <summary> Resets this stream to the beginning. This is an optional operation, so
+ /// subclasses may or may not implement this method. <see cref="Reset()" /> is not needed for
+ /// the standard indexing process. However, if the tokens of a
+ /// <c>TokenStream</c> are intended to be consumed more than once, it is
+ /// necessary to implement <see cref="Reset()" />. Note that if your TokenStream
+ /// caches tokens and feeds them back again after a reset, it is imperative
+ /// that you clone the tokens when you store them away (on the first pass) as
+ /// well as when you return them (on future passes after <see cref="Reset()" />).
+ /// </summary>
+ public virtual void Reset()
+ {
+ }
+
+ /// <summary>Releases resources associated with this stream. </summary>
+ [Obsolete("Use Dispose() instead")]
+ public void Close()
+ {
Dispose();
- }
+ }
public void Dispose()
{
Dispose(true);
}
- protected abstract void Dispose(bool disposing);
- }
+ protected abstract void Dispose(bool disposing);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Tokenattributes/FlagsAttribute.cs b/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
index b5c4b7b..afff36d 100644
--- a/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
+++ b/src/core/Analysis/Tokenattributes/FlagsAttribute.cs
@@ -20,66 +20,66 @@ using Attribute = Lucene.Net.Util.Attribute;
namespace Lucene.Net.Analysis.Tokenattributes
{
-
- /// <summary> This attribute can be used to pass different flags down the tokenizer chain,
- /// eg from one TokenFilter to another one.
- /// </summary>
- [Serializable]
- public class FlagsAttribute:Util.Attribute, IFlagsAttribute, System.ICloneable
- {
- private int flags = 0;
+
+ /// <summary> This attribute can be used to pass different flags down the tokenizer chain,
+ /// eg from one TokenFilter to another one.
+ /// </summary>
+ [Serializable]
+ public class FlagsAttribute:Util.Attribute, IFlagsAttribute, System.ICloneable
+ {
+ private int flags = 0;
- /// <summary> EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
- /// <p/>
- ///
- /// Get the bitset for any bits that have been set. This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
- /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
- ///
- ///
- /// </summary>
- /// <value> The bits </value>
- public virtual int Flags
- {
- get { return flags; }
- set { this.flags = value; }
- }
+ /// <summary> EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
+ /// <p/>
+ ///
+ /// Get the bitset for any bits that have been set. This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
+ /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
+ ///
+ ///
+ /// </summary>
+ /// <value> The bits </value>
+ public virtual int Flags
+ {
+ get { return flags; }
+ set { this.flags = value; }
+ }
- public override void Clear()
- {
- flags = 0;
- }
-
- public override bool Equals(System.Object other)
- {
- if (this == other)
- {
- return true;
- }
-
- if (other is FlagsAttribute)
- {
- return ((FlagsAttribute) other).flags == flags;
- }
-
- return false;
- }
-
- public override int GetHashCode()
- {
- return flags;
- }
-
- public override void CopyTo(Attribute target)
- {
- IFlagsAttribute t = (IFlagsAttribute) target;
- t.Flags = flags;
- }
-
- override public System.Object Clone()
- {
+ public override void Clear()
+ {
+ flags = 0;
+ }
+
+ public override bool Equals(System.Object other)
+ {
+ if (this == other)
+ {
+ return true;
+ }
+
+ if (other is FlagsAttribute)
+ {
+ return ((FlagsAttribute) other).flags == flags;
+ }
+
+ return false;
+ }
+
+ public override int GetHashCode()
+ {
+ return flags;
+ }
+
+ public override void CopyTo(Attribute target)
+ {
+ IFlagsAttribute t = (IFlagsAttribute) target;
+ t.Flags = flags;
+ }
+
+ override public System.Object Clone()
+ {
FlagsAttribute impl = new FlagsAttribute();
impl.flags = this.flags;
return impl;
- }
- }
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs b/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
index 24b2bea..325038e 100644
--- a/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
+++ b/src/core/Analysis/Tokenattributes/IFlagsAttribute.cs
@@ -21,21 +21,21 @@ using Tokenizer = Lucene.Net.Analysis.Tokenizer;
namespace Lucene.Net.Analysis.Tokenattributes
{
-
- /// <summary> This attribute can be used to pass different flags down the <see cref="Tokenizer" /> chain,
- /// eg from one TokenFilter to another one.
- /// </summary>
- public interface IFlagsAttribute:IAttribute
- {
- /// <summary> EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
- /// <p/>
- ///
- /// Get the bitset for any bits that have been set. This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
- /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
- ///
- ///
- /// </summary>
- /// <value> The bits </value>
- int Flags { get; set; }
- }
+
+ /// <summary> This attribute can be used to pass different flags down the <see cref="Tokenizer" /> chain,
+ /// eg from one TokenFilter to another one.
+ /// </summary>
+ public interface IFlagsAttribute:IAttribute
+ {
+ /// <summary> EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long.
+ /// <p/>
+ ///
+ /// Get the bitset for any bits that have been set. This is completely distinct from <see cref="ITypeAttribute.Type()" />, although they do share similar purposes.
+ /// The flags can be used to encode information about the token for use by other <see cref="Lucene.Net.Analysis.TokenFilter" />s.
+ ///
+ ///
+ /// </summary>
+ /// <value> The bits </value>
+ int Flags { get; set; }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/62f018ab/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
----------------------------------------------------------------------
diff --git a/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs b/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
index ffbbe02..3b00369 100644
--- a/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
+++ b/src/core/Analysis/Tokenattributes/IOffsetAttribute.cs
@@ -20,29 +20,29 @@ using Lucene.Net.Util;
namespace Lucene.Net.Analysis.Tokenattributes
{
-
- /// <summary> The start and end character offset of a Token. </summary>
- public interface IOffsetAttribute : IAttribute
- {
- /// <summary>Returns this Token's starting offset, the position of the first character
- /// corresponding to this token in the source text.
- /// Note that the difference between endOffset() and startOffset() may not be
- /// equal to termText.length(), as the term text may have been altered by a
- /// stemmer or some other filter.
- /// </summary>
- int StartOffset { get; }
+
+ /// <summary> The start and end character offset of a Token. </summary>
+ public interface IOffsetAttribute : IAttribute
+ {
+ /// <summary>Returns this Token's starting offset, the position of the first character
+ /// corresponding to this token in the source text.
+ /// Note that the difference between endOffset() and startOffset() may not be
+ /// equal to termText.length(), as the term text may have been altered by a
+ /// stemmer or some other filter.
+ /// </summary>
+ int StartOffset { get; }
- /// <summary>Set the starting and ending offset.
+ /// <summary>Set the starting and ending offset.
/// See StartOffset() and EndOffset()
/// </summary>
- void SetOffset(int startOffset, int endOffset);
+ void SetOffset(int startOffset, int endOffset);
- /// <summary>Returns this Token's ending offset, one greater than the position of the
- /// last character corresponding to this token in the source text. The length
- /// of the token in the source text is (endOffset - startOffset).
- /// </summary>
- int EndOffset { get; }
- }
+ /// <summary>Returns this Token's ending offset, one greater than the position of the
+ /// last character corresponding to this token in the source text. The length
+ /// of the token in the source text is (endOffset - startOffset).
+ /// </summary>
+ int EndOffset { get; }
+ }
}
\ No newline at end of file