You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/03/15 23:50:19 UTC

[37/50] [abbrv] lucenenet git commit: Lucene.Net.Analysis.Common.Analysis.Miscellaneous (CapitalizationFilter + CapitalizationFilterFactory): Changed implementation to consistently use culture-sensitive casing (unlike the way Lucene did), and added const

Lucene.Net.Analysis.Common.Analysis.Miscellaneous (CapitalizationFilter + CapitalizationFilterFactory): Changed implementation to consistently use culture-sensitive casing (unlike the way Lucene did), and added constructor overloads that allow you to specify a specific culture.


Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/7e7b44ed
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/7e7b44ed
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/7e7b44ed

Branch: refs/heads/api-work
Commit: 7e7b44ed4de17fa025223ef559faac11374a7771
Parents: 26d6c3f
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Wed Mar 15 20:51:05 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Wed Mar 15 21:49:43 2017 +0700

----------------------------------------------------------------------
 .../Miscellaneous/CapitalizationFilter.cs       | 75 +++++++++++++++++---
 .../CapitalizationFilterFactory.cs              | 11 +--
 .../Miscellaneous/TestCapitalizationFilter.cs   |  6 +-
 .../TestCapitalizationFilterFactory.cs          |  5 +-
 4 files changed, 81 insertions(+), 16 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
index 1148aca..2eaaf54 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
@@ -46,21 +46,44 @@ namespace Lucene.Net.Analysis.Miscellaneous
 
         private readonly ICharTermAttribute termAtt;
 
+        // LUCENENET specific for specifying culture instead of using
+        // invariant culture (which makes this class more generally useful).
+        // Per MSDN, InvariantCulture shouldn't be used for cases such as this:
+        // https://msdn.microsoft.com/en-us/library/dd465121(v=vs.110).aspx
+        private readonly CultureInfo culture;
+
         /// <summary>
-        /// Creates a <see cref="CapitalizationFilter"/> with the default parameters.
+        /// Creates a <see cref="CapitalizationFilter"/> with the default parameters using the culture from the current thread.
         /// <para>
         /// Calls <see cref="CapitalizationFilter.CapitalizationFilter(TokenStream, bool, CharArraySet, bool, ICollection{char[]}, int, int, int)">
-        ///     CapitalizationFilter(in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH)
+        ///     CapitalizationFilter(in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH, null)
         /// </see>
         /// </para>
         /// </summary>
         public CapitalizationFilter(TokenStream @in)
-            : this(@in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH)
+            : this(@in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH, null)
+        {
+        }
+
+        /// <summary>
+        /// Creates a <see cref="CapitalizationFilter"/> with the default parameters and the specified <paramref name="culture"/>.
+        /// <para>
+        /// Calls <see cref="CapitalizationFilter.CapitalizationFilter(TokenStream, bool, CharArraySet, bool, ICollection{char[]}, int, int, int)">
+        ///     CapitalizationFilter(in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH)
+        /// </see>
+        /// </para>
+        /// </summary>
+        /// <param name="in"> input tokenstream </param>
+        /// <param name="culture"> The culture to use for the casing operation. If null, the culture of the current thread will be used. </param>
+        // LUCENENET specific overload for specifying culture instead of using
+        // invariant culture (which makes this class more generally useful).
+        public CapitalizationFilter(TokenStream @in, CultureInfo culture)
+            : this(@in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH, culture)
         {
         }
 
         /// <summary>
-        /// Creates a <see cref="CapitalizationFilter"/> with the specified parameters. </summary>
+        /// Creates a <see cref="CapitalizationFilter"/> with the specified parameters using the culture from the current thread.</summary>
         /// <param name="in"> input tokenstream </param>
         /// <param name="onlyFirstWord"> should each word be capitalized or all of the words? </param>
         /// <param name="keep"> a keep word list.  Each word that should be kept separated by whitespace. </param>
@@ -70,8 +93,28 @@ namespace Lucene.Net.Analysis.Miscellaneous
         ///                      minWordLength is 3, "and" &gt; "And" but "or" stays "or". </param>
         /// <param name="maxWordCount"> if the token contains more then maxWordCount words, the capitalization is
         ///                     assumed to be correct. </param>
-        /// <param name="maxTokenLength"> ??? </param>
+        /// <param name="maxTokenLength"> The maximum length for an individual token. Tokens that exceed this length will not have the capitalization operation performed. </param>
         public CapitalizationFilter(TokenStream @in, bool onlyFirstWord, CharArraySet keep, bool forceFirstLetter, ICollection<char[]> okPrefix, int minWordLength, int maxWordCount, int maxTokenLength)
+            : this(@in, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength, null)
+        {
+        }
+
+        /// <summary>
+        /// Creates a <see cref="CapitalizationFilter"/> with the specified parameters and the specified <paramref name="culture"/>. </summary>
+        /// <param name="in"> input tokenstream </param>
+        /// <param name="onlyFirstWord"> should each word be capitalized or all of the words? </param>
+        /// <param name="keep"> a keep word list.  Each word that should be kept separated by whitespace. </param>
+        /// <param name="forceFirstLetter"> Force the first letter to be capitalized even if it is in the keep list. </param>
+        /// <param name="okPrefix"> do not change word capitalization if a word begins with something in this list. </param>
+        /// <param name="minWordLength"> how long the word needs to be to get capitalization applied.  If the
+        ///                      minWordLength is 3, "and" &gt; "And" but "or" stays "or". </param>
+        /// <param name="maxWordCount"> if the token contains more then maxWordCount words, the capitalization is
+        ///                     assumed to be correct. </param>
+        /// <param name="maxTokenLength"> The maximum length for an individual token. Tokens that exceed this length will not have the capitalization operation performed. </param>
+        /// <param name="culture"> The culture to use for the casing operation. If null, the culture of the current thread will be used. </param>
+        // LUCENENET specific overload for specifying culture instead of using
+        // invariant culture (which makes this class more generally useful).
+        public CapitalizationFilter(TokenStream @in, bool onlyFirstWord, CharArraySet keep, bool forceFirstLetter, ICollection<char[]> okPrefix, int minWordLength, int maxWordCount, int maxTokenLength, CultureInfo culture)
             : base(@in)
         {
             // LUCENENET NOTE: The guard clauses were copied here from a later version of Lucene.
@@ -97,6 +140,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
             this.minWordLength = minWordLength;
             this.maxWordCount = maxWordCount;
             this.maxTokenLength = maxTokenLength;
+            this.culture = culture;
             termAtt = AddAttribute<ICharTermAttribute>();
         }
 
@@ -164,8 +208,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
             {
                 for (int i = 0; i < length; i++)
                 {
-                    buffer[offset + i] = char.ToLower(buffer[offset + i]);
-
+                    buffer[offset + i] = Culture.TextInfo.ToLower(buffer[offset + i]);
                 }
                 return;
             }
@@ -174,7 +217,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
             {
                 if (wordCount == 0 && forceFirstLetter)
                 {
-                    buffer[offset] = CultureInfo.InvariantCulture.TextInfo.ToUpper(buffer[offset]);
+                    buffer[offset] = Culture.TextInfo.ToUpper(buffer[offset]);
                 }
                 return;
             }
@@ -211,13 +254,25 @@ namespace Lucene.Net.Analysis.Miscellaneous
             /*char[] chars = w.toCharArray();
             StringBuilder word = new StringBuilder( w.length() );
             word.append( Character.toUpperCase( chars[0] ) );*/
-            buffer[offset] = char.ToUpper(buffer[offset]);
+            buffer[offset] = Culture.TextInfo.ToUpper(buffer[offset]);
 
             for (int i = 1; i < length; i++)
             {
-                buffer[offset + i] = CultureInfo.InvariantCulture.TextInfo.ToLower(buffer[offset + i]);
+                buffer[offset + i] = Culture.TextInfo.ToLower(buffer[offset + i]);
             }
             //return word.toString();
         }
+
+        // LUCENENET specific - we get the culture from the current thread if it wasn't
+        // provided in the constructor. This allows us to change the culture on the current
+        // thread and work like other .NET components. But culture can be overridden by
+        // passing it to the constructor.
+        private CultureInfo Culture
+        {
+            get
+            {
+                return (culture != null) ? culture : CultureInfo.CurrentCulture;
+            }
+        }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
index 9af721e..3b61583 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
@@ -1,6 +1,6 @@
-\ufeffusing System.Collections.Generic;
-using Lucene.Net.Analysis.Util;
-using Lucene.Net.Support;
+\ufeffusing Lucene.Net.Analysis.Util;
+using System.Collections.Generic;
+using System.Globalization;
 
 namespace Lucene.Net.Analysis.Miscellaneous
 {
@@ -59,6 +59,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
         public const string MAX_TOKEN_LENGTH = "maxTokenLength";
         public const string ONLY_FIRST_WORD = "onlyFirstWord";
         public const string FORCE_FIRST_LETTER = "forceFirstLetter";
+        public const string CULTURE = "culture"; // LUCENENET specific
 
         internal CharArraySet keep;
 
@@ -69,6 +70,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
         internal readonly int maxTokenLength;
         internal readonly bool onlyFirstWord;
         internal readonly bool forceFirstLetter; // make sure the first letter is capital even if it is in the keep list
+        private readonly CultureInfo culture; // LUCENENET specific
 
         /// <summary>
         /// Creates a new <see cref="CapitalizationFilterFactory"/> </summary>
@@ -99,6 +101,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
             maxTokenLength = GetInt32(args, MAX_TOKEN_LENGTH, CapitalizationFilter.DEFAULT_MAX_TOKEN_LENGTH);
             onlyFirstWord = GetBoolean(args, ONLY_FIRST_WORD, true);
             forceFirstLetter = GetBoolean(args, FORCE_FIRST_LETTER, true);
+            culture = GetCulture(args, CULTURE, null);
             if (args.Count > 0)
             {
                 throw new System.ArgumentException("Unknown parameters: " + args);
@@ -107,7 +110,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
 
         public override TokenStream Create(TokenStream input)
         {
-            return new CapitalizationFilter(input, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength);
+            return new CapitalizationFilter(input, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength, culture);
         }
     }
 }
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
index c72219c..a95c131 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
@@ -4,6 +4,7 @@ using Lucene.Net.Support;
 using NUnit.Framework;
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 
 namespace Lucene.Net.Analysis.Miscellaneous
@@ -72,7 +73,10 @@ namespace Lucene.Net.Analysis.Miscellaneous
 
         internal static void AssertCapitalizesTo(Tokenizer tokenizer, string[] expected, bool onlyFirstWord, CharArraySet keep, bool forceFirstLetter, ICollection<char[]> okPrefix, int minWordLength, int maxWordCount, int maxTokenLength)
         {
-            CapitalizationFilter filter = new CapitalizationFilter(tokenizer, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength);
+            CapitalizationFilter filter = new CapitalizationFilter(tokenizer, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength,
+                // LUCENENET specific - pass in the invariant culture to get the same behavior as Lucene,
+                // otherwise the filter is culture-sensitive.
+                CultureInfo.InvariantCulture);
             AssertTokenStreamContents(filter, expected);
         }
 

http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
index 497ee45..15a5076 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
@@ -95,7 +95,10 @@ namespace Lucene.Net.Analysis.Miscellaneous
         {
             Reader reader = new StringReader("Hello thEre my Name is Ryan");
             TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
-            stream = TokenFilterFactory("Capitalization", "keep", "and the it BIG", "onlyFirstWord", "false", "forceFirstLetter", "true").Create(stream);
+            stream = TokenFilterFactory("Capitalization", "keep", "and the it BIG", "onlyFirstWord", "false", "forceFirstLetter", "true", 
+                // LUCENENET specific - pass in the invariant culture to get the same behavior as Lucene,
+                // otherwise the filter is culture-sensitive.
+                "culture", "invariant").Create(stream);
             AssertTokenStreamContents(stream, new string[] { "Hello", "There", "My", "Name", "Is", "Ryan" });
         }