You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2017/03/15 23:50:19 UTC
[37/50] [abbrv] lucenenet git commit:
Lucene.Net.Analysis.Common.Analysis.Miscellaneous (CapitalizationFilter +
CapitalizationFilterFactory): Changed implementation to consistently use
culture-sensitive casing (unlike the way Lucene did), and added const
Lucene.Net.Analysis.Common.Analysis.Miscellaneous (CapitalizationFilter + CapitalizationFilterFactory): Changed implementation to consistently use culture-sensitive casing (unlike the way Lucene did), and added constructor overloads that allow you to specify a specific culture.
Project: http://git-wip-us.apache.org/repos/asf/lucenenet/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucenenet/commit/7e7b44ed
Tree: http://git-wip-us.apache.org/repos/asf/lucenenet/tree/7e7b44ed
Diff: http://git-wip-us.apache.org/repos/asf/lucenenet/diff/7e7b44ed
Branch: refs/heads/api-work
Commit: 7e7b44ed4de17fa025223ef559faac11374a7771
Parents: 26d6c3f
Author: Shad Storhaug <sh...@shadstorhaug.com>
Authored: Wed Mar 15 20:51:05 2017 +0700
Committer: Shad Storhaug <sh...@shadstorhaug.com>
Committed: Wed Mar 15 21:49:43 2017 +0700
----------------------------------------------------------------------
.../Miscellaneous/CapitalizationFilter.cs | 75 +++++++++++++++++---
.../CapitalizationFilterFactory.cs | 11 +--
.../Miscellaneous/TestCapitalizationFilter.cs | 6 +-
.../TestCapitalizationFilterFactory.cs | 5 +-
4 files changed, 81 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
index 1148aca..2eaaf54 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilter.cs
@@ -46,21 +46,44 @@ namespace Lucene.Net.Analysis.Miscellaneous
private readonly ICharTermAttribute termAtt;
+ // LUCENENET specific for specifying culture instead of using
+ // invariant culture (which makes this class more generally useful).
+ // Per MSDN, InvariantCulture shouldn't be used for cases such as this:
+ // https://msdn.microsoft.com/en-us/library/dd465121(v=vs.110).aspx
+ private readonly CultureInfo culture;
+
/// <summary>
- /// Creates a <see cref="CapitalizationFilter"/> with the default parameters.
+ /// Creates a <see cref="CapitalizationFilter"/> with the default parameters using the culture from the current thread.
/// <para>
/// Calls <see cref="CapitalizationFilter.CapitalizationFilter(TokenStream, bool, CharArraySet, bool, ICollection{char[]}, int, int, int)">
- /// CapitalizationFilter(in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH)
+ /// CapitalizationFilter(in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH, null)
/// </see>
/// </para>
/// </summary>
public CapitalizationFilter(TokenStream @in)
- : this(@in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH)
+ : this(@in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH, null)
+ {
+ }
+
+ /// <summary>
+ /// Creates a <see cref="CapitalizationFilter"/> with the default parameters and the specified <paramref name="culture"/>.
+ /// <para>
+ /// Calls <see cref="CapitalizationFilter.CapitalizationFilter(TokenStream, bool, CharArraySet, bool, ICollection{char[]}, int, int, int)">
+ /// CapitalizationFilter(in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH)
+ /// </see>
+ /// </para>
+ /// </summary>
+ /// <param name="in"> input tokenstream </param>
+ /// <param name="culture"> The culture to use for the casing operation. If null, the culture of the current thread will be used. </param>
+ // LUCENENET specific overload for specifying culture instead of using
+ // invariant culture (which makes this class more generally useful).
+ public CapitalizationFilter(TokenStream @in, CultureInfo culture)
+ : this(@in, true, null, true, null, 0, DEFAULT_MAX_WORD_COUNT, DEFAULT_MAX_TOKEN_LENGTH, culture)
{
}
/// <summary>
- /// Creates a <see cref="CapitalizationFilter"/> with the specified parameters. </summary>
+ /// Creates a <see cref="CapitalizationFilter"/> with the specified parameters using the culture from the current thread.</summary>
/// <param name="in"> input tokenstream </param>
/// <param name="onlyFirstWord"> should each word be capitalized or all of the words? </param>
/// <param name="keep"> a keep word list. Each word that should be kept separated by whitespace. </param>
@@ -70,8 +93,28 @@ namespace Lucene.Net.Analysis.Miscellaneous
/// minWordLength is 3, "and" > "And" but "or" stays "or". </param>
/// <param name="maxWordCount"> if the token contains more then maxWordCount words, the capitalization is
/// assumed to be correct. </param>
- /// <param name="maxTokenLength"> ??? </param>
+ /// <param name="maxTokenLength"> The maximum length for an individual token. Tokens that exceed this length will not have the capitalization operation performed. </param>
public CapitalizationFilter(TokenStream @in, bool onlyFirstWord, CharArraySet keep, bool forceFirstLetter, ICollection<char[]> okPrefix, int minWordLength, int maxWordCount, int maxTokenLength)
+ : this(@in, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength, null)
+ {
+ }
+
+ /// <summary>
+ /// Creates a <see cref="CapitalizationFilter"/> with the specified parameters and the specified <paramref name="culture"/>. </summary>
+ /// <param name="in"> input tokenstream </param>
+ /// <param name="onlyFirstWord"> should each word be capitalized or all of the words? </param>
+ /// <param name="keep"> a keep word list. Each word that should be kept separated by whitespace. </param>
+ /// <param name="forceFirstLetter"> Force the first letter to be capitalized even if it is in the keep list. </param>
+ /// <param name="okPrefix"> do not change word capitalization if a word begins with something in this list. </param>
+ /// <param name="minWordLength"> how long the word needs to be to get capitalization applied. If the
+ /// minWordLength is 3, "and" > "And" but "or" stays "or". </param>
+ /// <param name="maxWordCount"> if the token contains more then maxWordCount words, the capitalization is
+ /// assumed to be correct. </param>
+ /// <param name="maxTokenLength"> The maximum length for an individual token. Tokens that exceed this length will not have the capitalization operation performed. </param>
+ /// <param name="culture"> The culture to use for the casing operation. If null, the culture of the current thread will be used. </param>
+ // LUCENENET specific overload for specifying culture instead of using
+ // invariant culture (which makes this class more generally useful).
+ public CapitalizationFilter(TokenStream @in, bool onlyFirstWord, CharArraySet keep, bool forceFirstLetter, ICollection<char[]> okPrefix, int minWordLength, int maxWordCount, int maxTokenLength, CultureInfo culture)
: base(@in)
{
// LUCENENET NOTE: The guard clauses were copied here from a later version of Lucene.
@@ -97,6 +140,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
this.minWordLength = minWordLength;
this.maxWordCount = maxWordCount;
this.maxTokenLength = maxTokenLength;
+ this.culture = culture;
termAtt = AddAttribute<ICharTermAttribute>();
}
@@ -164,8 +208,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
{
for (int i = 0; i < length; i++)
{
- buffer[offset + i] = char.ToLower(buffer[offset + i]);
-
+ buffer[offset + i] = Culture.TextInfo.ToLower(buffer[offset + i]);
}
return;
}
@@ -174,7 +217,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
{
if (wordCount == 0 && forceFirstLetter)
{
- buffer[offset] = CultureInfo.InvariantCulture.TextInfo.ToUpper(buffer[offset]);
+ buffer[offset] = Culture.TextInfo.ToUpper(buffer[offset]);
}
return;
}
@@ -211,13 +254,25 @@ namespace Lucene.Net.Analysis.Miscellaneous
/*char[] chars = w.toCharArray();
StringBuilder word = new StringBuilder( w.length() );
word.append( Character.toUpperCase( chars[0] ) );*/
- buffer[offset] = char.ToUpper(buffer[offset]);
+ buffer[offset] = Culture.TextInfo.ToUpper(buffer[offset]);
for (int i = 1; i < length; i++)
{
- buffer[offset + i] = CultureInfo.InvariantCulture.TextInfo.ToLower(buffer[offset + i]);
+ buffer[offset + i] = Culture.TextInfo.ToLower(buffer[offset + i]);
}
//return word.toString();
}
+
+ // LUCENENET specific - we get the culture from the current thread if it wasn't
+ // provided in the constructor. This allows us to change the culture on the current
+ // thread and work like other .NET components. But culture can be overridden by
+ // passing it to the constructor.
+ private CultureInfo Culture
+ {
+ get
+ {
+ return (culture != null) ? culture : CultureInfo.CurrentCulture;
+ }
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
index 9af721e..3b61583 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Miscellaneous/CapitalizationFilterFactory.cs
@@ -1,6 +1,6 @@
-\ufeffusing System.Collections.Generic;
-using Lucene.Net.Analysis.Util;
-using Lucene.Net.Support;
+\ufeffusing Lucene.Net.Analysis.Util;
+using System.Collections.Generic;
+using System.Globalization;
namespace Lucene.Net.Analysis.Miscellaneous
{
@@ -59,6 +59,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
public const string MAX_TOKEN_LENGTH = "maxTokenLength";
public const string ONLY_FIRST_WORD = "onlyFirstWord";
public const string FORCE_FIRST_LETTER = "forceFirstLetter";
+ public const string CULTURE = "culture"; // LUCENENET specific
internal CharArraySet keep;
@@ -69,6 +70,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
internal readonly int maxTokenLength;
internal readonly bool onlyFirstWord;
internal readonly bool forceFirstLetter; // make sure the first letter is capital even if it is in the keep list
+ private readonly CultureInfo culture; // LUCENENET specific
/// <summary>
/// Creates a new <see cref="CapitalizationFilterFactory"/> </summary>
@@ -99,6 +101,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
maxTokenLength = GetInt32(args, MAX_TOKEN_LENGTH, CapitalizationFilter.DEFAULT_MAX_TOKEN_LENGTH);
onlyFirstWord = GetBoolean(args, ONLY_FIRST_WORD, true);
forceFirstLetter = GetBoolean(args, FORCE_FIRST_LETTER, true);
+ culture = GetCulture(args, CULTURE, null);
if (args.Count > 0)
{
throw new System.ArgumentException("Unknown parameters: " + args);
@@ -107,7 +110,7 @@ namespace Lucene.Net.Analysis.Miscellaneous
public override TokenStream Create(TokenStream input)
{
- return new CapitalizationFilter(input, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength);
+ return new CapitalizationFilter(input, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength, culture);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
index c72219c..a95c131 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilter.cs
@@ -4,6 +4,7 @@ using Lucene.Net.Support;
using NUnit.Framework;
using System;
using System.Collections.Generic;
+using System.Globalization;
using System.IO;
namespace Lucene.Net.Analysis.Miscellaneous
@@ -72,7 +73,10 @@ namespace Lucene.Net.Analysis.Miscellaneous
internal static void AssertCapitalizesTo(Tokenizer tokenizer, string[] expected, bool onlyFirstWord, CharArraySet keep, bool forceFirstLetter, ICollection<char[]> okPrefix, int minWordLength, int maxWordCount, int maxTokenLength)
{
- CapitalizationFilter filter = new CapitalizationFilter(tokenizer, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength);
+ CapitalizationFilter filter = new CapitalizationFilter(tokenizer, onlyFirstWord, keep, forceFirstLetter, okPrefix, minWordLength, maxWordCount, maxTokenLength,
+ // LUCENENET specific - pass in the invariant culture to get the same behavior as Lucene,
+ // otherwise the filter is culture-sensitive.
+ CultureInfo.InvariantCulture);
AssertTokenStreamContents(filter, expected);
}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/7e7b44ed/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
index 497ee45..15a5076 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Miscellaneous/TestCapitalizationFilterFactory.cs
@@ -95,7 +95,10 @@ namespace Lucene.Net.Analysis.Miscellaneous
{
Reader reader = new StringReader("Hello thEre my Name is Ryan");
TokenStream stream = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
- stream = TokenFilterFactory("Capitalization", "keep", "and the it BIG", "onlyFirstWord", "false", "forceFirstLetter", "true").Create(stream);
+ stream = TokenFilterFactory("Capitalization", "keep", "and the it BIG", "onlyFirstWord", "false", "forceFirstLetter", "true",
+ // LUCENENET specific - pass in the invariant culture to get the same behavior as Lucene,
+ // otherwise the filter is culture-sensitive.
+ "culture", "invariant").Create(stream);
AssertTokenStreamContents(stream, new string[] { "Hello", "There", "My", "Name", "Is", "Ryan" });
}