You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by pa...@apache.org on 2026/06/16 19:06:02 UTC
(lucenenet) branch master updated: Finish port of Lucene.Net.Collation functionality, #1343 (#1347)
This is an automated email from the ASF dual-hosted git repository.
paulirwin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
The following commit(s) were added to refs/heads/master by this push:
new ba8f0497a Finish port of Lucene.Net.Collation functionality, #1343 (#1347)
ba8f0497a is described below
commit ba8f0497ad5d8c63984c6859e2369ea37b660ece
Author: Paul Irwin <pa...@gmail.com>
AuthorDate: Tue Jun 16 13:05:56 2026 -0600
Finish port of Lucene.Net.Collation functionality, #1343 (#1347)
Co-authored-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
Directory.Build.targets | 2 +
.../Collation/CollationAttributeFactory.cs | 77 ++++++----
.../Collation/CollationKeyAnalyzer.cs | 70 +++++----
.../Collation/CollationKeyFilter.cs | 90 +++++------
.../Collation/CollationKeyFilterFactory.cs | 168 ++++++---------------
.../TokenAttributes/CollatedTermAttributeImpl.cs | 63 ++++++--
.../Collation/package.md | 135 ++++++++++-------
.../Support/Collation/CollationUtil.cs | 58 +++++++
.../ByTask/Tasks/NewCollationAnalyzerTask.cs | 47 +++---
.../Analysis/CollationTestBase.cs | 33 +++-
.../Collation/TestCollationKeyAnalyzer.cs | 107 +++----------
.../Collation/TestCollationKeyFilter.cs | 101 +++----------
.../Collation/TestCollationKeyFilterFactory.cs | 63 ++------
.../ByTask/TestPerfTasksLogic.cs | 28 +++-
14 files changed, 507 insertions(+), 535 deletions(-)
diff --git a/Directory.Build.targets b/Directory.Build.targets
index 2de12f6fe..fe37bc9e7 100644
--- a/Directory.Build.targets
+++ b/Directory.Build.targets
@@ -77,6 +77,8 @@
<DefineConstants>$(DefineConstants);FEATURE_MEMORYMARSHAL_GETARRAYDATAREFERENCE</DefineConstants>
<DefineConstants>$(DefineConstants);FEATURE_READONLYSET</DefineConstants>
+ <!-- CompareInfo.GetSortKey(ReadOnlySpan<char>, Span<byte>, CompareOptions) and GetSortKeyLength(ReadOnlySpan<char>, CompareOptions) -->
+ <DefineConstants>$(DefineConstants);FEATURE_COMPAREINFO_GETSORTKEY_SPAN</DefineConstants>
</PropertyGroup>
diff --git a/src/Lucene.Net.Analysis.Common/Collation/CollationAttributeFactory.cs b/src/Lucene.Net.Analysis.Common/Collation/CollationAttributeFactory.cs
index f65e0fb60..7d271adba 100644
--- a/src/Lucene.Net.Analysis.Common/Collation/CollationAttributeFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Collation/CollationAttributeFactory.cs
@@ -1,9 +1,7 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using Icu.Collation;
using Lucene.Net.Collation.TokenAttributes;
using Lucene.Net.Util;
-using System.Reflection;
+using System.Globalization;
namespace Lucene.Net.Collation
{
@@ -30,47 +28,44 @@ namespace Lucene.Net.Collation
/// encodes the bytes as an index term.
/// </para>
/// <para>
- /// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="Collator"/> at
+ /// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="CompareInfo"/> at
/// index and query time -- <see cref="System.Globalization.SortKey"/>s are only comparable when produced by
- /// the same <see cref="Collator"/>. Since <see cref="RuleBasedCollator"/>s are not
+ /// the same <see cref="CompareInfo"/>. Since the platform collator is not
/// independently versioned, it is unsafe to search against stored
/// <see cref="System.Globalization.SortKey"/>s unless the following are exactly the same (best practice is
/// to store this information with the index and check that they remain the
/// same at query time):
/// </para>
/// <list type="number">
- /// <item><description>JVM vendor</description></item>
- /// <item><description>JVM version, including patch version</description></item>
+ /// <item><description>The .NET runtime version</description></item>
/// <item><description>
- /// The language (and country and variant, if specified) of the Locale
- /// used when constructing the collator via
- /// <see cref="Collator.Create(System.Globalization.CultureInfo)"/>.
+ /// The <see cref="CultureInfo"/> of the <see cref="CompareInfo"/> obtained via
+ /// <see cref="CompareInfo.GetCompareInfo(string)"/>.
/// </description></item>
/// <item><description>
- /// The collation strength used - see <see cref="Collator.Strength"/>
+ /// The <see cref="CompareOptions"/> used - which control the collation strength
+ /// and Unicode normalization.
/// </description></item>
/// </list>
/// <para>
- /// The <c>ICUCollationAttributeFactory</c> in the analysis-icu package
- /// uses ICU4J's Collator, which makes its
+ /// The <c>ICUCollationAttributeFactory</c> in the Lucene.Net.ICU package
+ /// uses ICU4N's Collator, which makes its
/// version available, thus allowing collation to be versioned independently
- /// from the JVM. ICUCollationAttributeFactory is also significantly faster and
- /// generates significantly shorter keys than CollationAttributeFactory. See
- /// <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
- /// >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
- /// generation timing and key length comparisons between ICU4J and
- /// <see cref="Collator"/> over several languages.
+ /// from the .NET runtime. ICUCollationAttributeFactory generates
+ /// significantly shorter keys than <see cref="CollationAttributeFactory"/> and exposes
+ /// tailoring/custom rules that the platform collator does not support.
/// </para>
/// <para>
- /// CollationKeys generated by java.text.Collators are not compatible
- /// with those those generated by ICU Collators. Specifically, if you use
- /// CollationAttributeFactory to generate index terms, do not use
+ /// <see cref="System.Globalization.SortKey"/>s generated by the platform collator are not compatible
+ /// with those generated by ICU Collators. Specifically, if you use
+ /// <see cref="CollationAttributeFactory"/> to generate index terms, do not use
/// ICUCollationAttributeFactory on the query side, or vice versa.
/// </para>
/// </summary>
public class CollationAttributeFactory : AttributeFactory
{
- private readonly Collator collator;
+ private readonly CompareInfo collator;
+ private readonly CompareOptions options;
private readonly AttributeFactory @delegate;
/// <summary>
@@ -78,8 +73,30 @@ public class CollationAttributeFactory : AttributeFactory
/// <see cref="AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY"/> as the
/// factory for all other attributes. </summary>
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
- public CollationAttributeFactory(Collator collator)
- : this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator)
+ public CollationAttributeFactory(CompareInfo collator)
+ : this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator, CompareOptions.None)
+ {
+ }
+
+ /// <summary>
+ /// Create a <see cref="CollationAttributeFactory"/>, using
+ /// <see cref="AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY"/> as the
+ /// factory for all other attributes. </summary>
+ /// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
+ /// <param name="options"> Collation options that control the collation strength and
+ /// Unicode normalization (decomposition) of the generated sort key. </param>
+ public CollationAttributeFactory(CompareInfo collator, CompareOptions options)
+ : this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, collator, options)
+ {
+ }
+
+ /// <summary>
+ /// Create a <see cref="CollationAttributeFactory"/>, using the supplied Attribute Factory
+ /// as the factory for all other attributes. </summary>
+ /// <param name="delegate"> Attribute Factory </param>
+ /// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
+ public CollationAttributeFactory(AttributeFactory @delegate, CompareInfo collator)
+ : this(@delegate, collator, CompareOptions.None)
{
}
@@ -88,18 +105,20 @@ public CollationAttributeFactory(Collator collator)
/// as the factory for all other attributes. </summary>
/// <param name="delegate"> Attribute Factory </param>
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
- public CollationAttributeFactory(AttributeFactory @delegate, Collator collator)
+ /// <param name="options"> Collation options that control the collation strength and
+ /// Unicode normalization (decomposition) of the generated sort key. </param>
+ public CollationAttributeFactory(AttributeFactory @delegate, CompareInfo collator, CompareOptions options)
{
this.@delegate = @delegate;
this.collator = collator;
+ this.options = options;
}
public override Attribute CreateAttributeInstance<T>()
{
- return typeof(T).IsAssignableFrom(typeof(CollatedTermAttributeImpl))
- ? new CollatedTermAttributeImpl(this.collator)
+ return typeof(T).IsAssignableFrom(typeof(CollatedTermAttribute))
+ ? new CollatedTermAttribute(this.collator, this.options)
: this.@delegate.CreateAttributeInstance<T>();
}
}
}
-#endif
diff --git a/src/Lucene.Net.Analysis.Common/Collation/CollationKeyAnalyzer.cs b/src/Lucene.Net.Analysis.Common/Collation/CollationKeyAnalyzer.cs
index 0dab5446d..6291f994c 100644
--- a/src/Lucene.Net.Analysis.Common/Collation/CollationKeyAnalyzer.cs
+++ b/src/Lucene.Net.Analysis.Common/Collation/CollationKeyAnalyzer.cs
@@ -1,10 +1,9 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using Icu.Collation;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Core;
using Lucene.Net.Util;
using System;
+using System.Globalization;
using System.IO;
namespace Lucene.Net.Collation
@@ -37,41 +36,37 @@ namespace Lucene.Net.Collation
/// it to be stored as an index term.
/// </para>
/// <para>
- /// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="Collator"/> at
+ /// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="CompareInfo"/> at
/// index and query time -- <see cref="System.Globalization.SortKey"/> are only comparable when produced by
- /// the same <see cref="Collator"/>. Since <c>java.text.RuleBasedCollators</c> are not
+ /// the same <see cref="CompareInfo"/>. Since the platform collator is not
/// independently versioned, it is unsafe to search against stored
/// <see cref="System.Globalization.SortKey"/> unless the following are exactly the same (best practice is
/// to store this information with the index and check that they remain the
/// same at query time):
/// </para>
/// <list type="number">
- /// <item><description>JVM vendor</description></item>
- /// <item><description>JVM version, including patch version</description></item>
+ /// <item><description>The .NET runtime version</description></item>
/// <item><description>
- /// The language (and country and variant, if specified) of the Locale
- /// used when constructing the collator via
- /// <see cref="Collator.Create(System.Globalization.CultureInfo)"/>.
+ /// The <see cref="CultureInfo"/> of the <see cref="CompareInfo"/> obtained via
+ /// <see cref="CompareInfo.GetCompareInfo(string)"/>.
/// </description></item>
/// <item><description>
- /// The collation strength used - see <see cref="Collator.Strength"/>
+ /// The <see cref="CompareOptions"/> used - which control the collation strength
+ /// and Unicode normalization.
/// </description></item>
/// </list>
/// <para>
- /// The <c>ICUCollationKeyAnalyzer</c> in the analysis-icu package
- /// uses ICU4J's Collator, which makes its
- /// its version available, thus allowing collation to be versioned
- /// independently from the JVM. ICUCollationKeyAnalyzer is also significantly
- /// faster and generates significantly shorter keys than CollationKeyAnalyzer.
- /// See <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
- /// >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
- /// generation timing and key length comparisons between ICU4J and
- /// <see cref="Collator"/> over several languages.
+ /// The <c>ICUCollationKeyAnalyzer</c> in the Lucene.Net.ICU package
+ /// uses ICU4N's Collator, which makes its
+ /// version available, thus allowing collation to be versioned
+ /// independently from the .NET runtime. ICUCollationKeyAnalyzer
+ /// generates significantly shorter keys than <see cref="CollationKeyAnalyzer"/> and
+ /// exposes tailoring/custom rules that the platform collator does not support.
/// </para>
/// <para>
- /// CollationKeys generated by <see cref="Collator"/> are not compatible
- /// with those those generated by ICU Collators. Specifically, if you use
- /// CollationKeyAnalyzer to generate index terms, do not use
+ /// <see cref="System.Globalization.SortKey"/>s generated by the platform collator are not compatible
+ /// with those generated by ICU Collators. Specifically, if you use
+ /// <see cref="CollationKeyAnalyzer"/> to generate index terms, do not use
/// ICUCollationKeyAnalyzer on the query side, or vice versa.
/// </para>
/// <para>You must specify the required <see cref="LuceneVersion"/>
@@ -84,7 +79,8 @@ namespace Lucene.Net.Collation
/// </summary>
public sealed class CollationKeyAnalyzer : Analyzer
{
- private readonly Collator collator;
+ private readonly CompareInfo collator;
+ private readonly CompareOptions options;
private readonly CollationAttributeFactory factory;
private readonly LuceneVersion matchVersion;
@@ -93,20 +89,33 @@ public sealed class CollationKeyAnalyzer : Analyzer
/// </summary>
/// <param name="matchVersion"> See <see cref="CollationKeyAnalyzer"/> </param>
/// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
- public CollationKeyAnalyzer(LuceneVersion matchVersion, Collator collator)
+ public CollationKeyAnalyzer(LuceneVersion matchVersion, CompareInfo collator)
+ : this(matchVersion, collator, CompareOptions.None)
+ {
+ }
+
+ /// <summary>
+ /// Create a new <see cref="CollationKeyAnalyzer"/>, using the specified collator.
+ /// </summary>
+ /// <param name="matchVersion"> See <see cref="CollationKeyAnalyzer"/> </param>
+ /// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
+ /// <param name="options"> Collation options that control the collation strength and
+ /// Unicode normalization (decomposition) of the generated sort key. </param>
+ public CollationKeyAnalyzer(LuceneVersion matchVersion, CompareInfo collator, CompareOptions options)
{
this.matchVersion = matchVersion;
this.collator = collator;
- this.factory = new CollationAttributeFactory(collator);
+ this.options = options;
+ this.factory = new CollationAttributeFactory(collator, options);
}
- [Obsolete("Use <seealso cref=\"CollationKeyAnalyzer#CollationKeyAnalyzer(LuceneVersion, Collator)\"/> and specify a version instead. This ctor will be removed in Lucene 5.0")]
- public CollationKeyAnalyzer(Collator collator)
- : this(LuceneVersion.LUCENE_31, collator)
+ [Obsolete("Use CollationKeyAnalyzer.CollationKeyAnalyzer(LuceneVersion, CompareInfo) and specify a version instead. This ctor will be removed in Lucene 5.0")]
+ public CollationKeyAnalyzer(CompareInfo collator)
+ : this(LuceneVersion.LUCENE_31, collator, CompareOptions.None)
{
}
- protected override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+ protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
#pragma warning disable 612, 618
if (this.matchVersion.OnOrAfter(LuceneVersion.LUCENE_40))
@@ -120,10 +129,9 @@ protected override TokenStreamComponents CreateComponents(string fieldName, Text
var tokenizer = new KeywordTokenizer(reader);
return new TokenStreamComponents(tokenizer,
#pragma warning disable 612, 618
- new CollationKeyFilter(tokenizer, this.collator));
+ new CollationKeyFilter(tokenizer, this.collator, this.options));
#pragma warning restore 612, 618
}
}
}
}
-#endif
diff --git a/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilter.cs b/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilter.cs
index 1c35101f0..b4666098d 100644
--- a/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilter.cs
@@ -1,11 +1,9 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using Icu.Collation;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.TokenAttributes;
-using Lucene.Net.Support;
using Lucene.Net.Util;
using System;
+using System.Globalization;
namespace Lucene.Net.Collation
{
@@ -33,41 +31,37 @@ namespace Lucene.Net.Collation
/// it to be stored as an index term.
/// </para>
/// <para>
- /// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="Collator"/> at
+ /// <strong>WARNING:</strong> Make sure you use exactly the same <see cref="CompareInfo"/> at
/// index and query time -- <see cref="System.Globalization.SortKey"/> are only comparable when produced by
- /// the same <see cref="Collator"/>. Since <c>java.text.RuleBasedCollators</c> are not
+ /// the same <see cref="CompareInfo"/>. Since the platform collator is not
/// independently versioned, it is unsafe to search against stored
/// <see cref="System.Globalization.SortKey"/> unless the following are exactly the same (best practice is
/// to store this information with the index and check that they remain the
/// same at query time):
/// </para>
/// <list type="number">
- /// <item><description>JVM vendor</description></item>
- /// <item><description>JVM version, including patch version</description></item>
+ /// <item><description>The .NET runtime version</description></item>
/// <item><description>
- /// The language (and country and variant, if specified) of the Locale
- /// used when constructing the collator via
- /// <see cref="Collator.Create(System.Globalization.CultureInfo)"/>.
+ /// The <see cref="CultureInfo"/> of the <see cref="CompareInfo"/> obtained via
+ /// <see cref="CompareInfo.GetCompareInfo(string)"/>.
/// </description></item>
/// <item><description>
- /// The collation strength used - see <see cref="Collator.Strength"/>
+ /// The <see cref="CompareOptions"/> used - which control the collation strength
+ /// and Unicode normalization.
/// </description></item>
/// </list>
/// <para>
- /// The <c>ICUCollationKeyFilter</c> in the analysis-icu package
- /// uses ICU4J's Collator, which makes its
+ /// The <c>ICUCollationKeyFilter</c> in the Lucene.Net.ICU package
+ /// uses ICU4N's Collator, which makes its
/// version available, thus allowing collation to be versioned independently
- /// from the JVM. ICUCollationKeyFilter is also significantly faster and
- /// generates significantly shorter keys than CollationKeyFilter. See
- /// <a href="http://site.icu-project.org/charts/collation-icu4j-sun"
- /// >http://site.icu-project.org/charts/collation-icu4j-sun</a> for key
- /// generation timing and key length comparisons between ICU4J and
- /// java.text.Collator over several languages.
+ /// from the .NET runtime. ICUCollationKeyFilter generates significantly
+ /// shorter keys than <see cref="CollationKeyFilter"/> and exposes tailoring/custom
+ /// rules that the platform collator does not support.
/// </para>
/// <para>
- /// CollationKeys generated by java.text.Collators are not compatible
- /// with those those generated by ICU Collators. Specifically, if you use
- /// CollationKeyFilter to generate index terms, do not use
+ /// <see cref="System.Globalization.SortKey"/>s generated by the platform collator are not compatible
+ /// with those generated by ICU Collators. Specifically, if you use
+ /// <see cref="CollationKeyFilter"/> to generate index terms, do not use
/// ICUCollationKeyFilter on the query side, or vice versa.
/// </para> </summary>
/// @deprecated Use <see cref="CollationAttributeFactory"/> instead, which encodes
@@ -75,15 +69,26 @@ namespace Lucene.Net.Collation
[Obsolete("Use CollationAttributeFactory instead, which encodes terms directly as bytes. This filter will be removed in Lucene 5.0.")]
public sealed class CollationKeyFilter : TokenFilter
{
- private readonly Collator collator;
+ private readonly CompareInfo collator;
+ private readonly CompareOptions options;
private readonly ICharTermAttribute termAtt;
/// <param name="input"> Source token stream </param>
- /// <param name="collator"> CollationKey generator </param>
- public CollationKeyFilter(TokenStream input, Collator collator)
+ /// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
+ public CollationKeyFilter(TokenStream input, CompareInfo collator)
+ : this(input, collator, CompareOptions.None)
+ {
+ }
+
+ /// <param name="input"> Source token stream </param>
+ /// <param name="collator"> <see cref="System.Globalization.SortKey"/> generator </param>
+ /// <param name="options"> Collation options that control the collation strength and
+ /// Unicode normalization (decomposition) of the generated sort key. </param>
+ public CollationKeyFilter(TokenStream input, CompareInfo collator, CompareOptions options)
: base(input)
{
this.collator = collator;
+ this.options = options;
this.termAtt = this.AddAttribute<ICharTermAttribute>();
}
@@ -91,8 +96,8 @@ public override bool IncrementToken()
{
if (this.m_input.IncrementToken())
{
- var collationKey = this.collator.GetSortKey(this.termAtt.ToString()).KeyData.ToSByteArray();
- var encodedLength = IndexableBinaryStringTools.GetEncodedLength(collationKey, 0, collationKey.Length);
+ byte[] collationKey = GetCollationKey();
+ int encodedLength = IndexableBinaryStringTools.GetEncodedLength(collationKey, 0, collationKey.Length);
this.termAtt.ResizeBuffer(encodedLength);
this.termAtt.Length = encodedLength;
@@ -104,23 +109,24 @@ public override bool IncrementToken()
return false;
}
}
- }
- internal static class ByteArrayExtentions
- {
- internal static byte[] ToByteArray(this sbyte[] arr)
+ private byte[] GetCollationKey()
{
- var unsigned = new byte[arr.Length];
- Arrays.Copy(arr, 0, unsigned, 0, arr.Length);
- return unsigned;
- }
-
- internal static sbyte[] ToSByteArray(this byte[] arr)
- {
- var unsigned = new sbyte[arr.Length];
- Arrays.Copy(arr, 0, unsigned, 0, arr.Length);
- return unsigned;
+#if FEATURE_COMPAREINFO_GETSORTKEY_SPAN
+ // LUCENENET: On .NET 5+ with ICU, CompareInfo has a ReadOnlySpan<char> overload of GetSortKey
+ // that lets us generate the sort key without converting the term to a string first. The NLS
+ // backend does not normalize internally (see CollatedTermAttribute.Normalize), so we still need
+ // the slow path when the app is configured for NLS.
+ if (CollationUtil.IsICU)
+ {
+ ReadOnlySpan<char> source = this.termAtt.AsSpan();
+ byte[] collationKey = new byte[this.collator.GetSortKeyLength(source, this.options)];
+ this.collator.GetSortKey(source, collationKey, this.options);
+ return collationKey;
+ }
+#endif
+ // LUCENENET: Normalize to NFC for cross-backend consistency. See CollatedTermAttribute.Normalize.
+ return this.collator.GetSortKey(TokenAttributes.CollatedTermAttribute.Normalize(this.termAtt.ToString()), this.options).KeyData;
}
}
}
-#endif
diff --git a/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilterFactory.cs b/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilterFactory.cs
index 88ca44c72..55ada5f4c 100644
--- a/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilterFactory.cs
+++ b/src/Lucene.Net.Analysis.Common/Collation/CollationKeyFilterFactory.cs
@@ -1,15 +1,9 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using Icu;
-using Icu.Collation;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Util;
-using Lucene.Net.Util;
using System;
using System.Collections.Generic;
using System.Globalization;
-using System.IO;
-using System.Text;
namespace Lucene.Net.Collation
{
@@ -33,14 +27,8 @@ namespace Lucene.Net.Collation
/// <summary>
/// Factory for <see cref="CollationKeyFilter"/>.
/// <para>
- /// This factory can be created in two ways:
- /// <list type="bullet">
- /// <item><description>Based upon a system collator associated with a <see cref="System.Globalization.CultureInfo"/>.</description></item>
- /// <item><description>Based upon a tailored ruleset.</description></item>
- /// </list>
- /// </para>
- /// <para>
- /// Using a System collator:
+ /// This factory is created based upon a system collator associated with a
+ /// <see cref="CultureInfo"/>:
/// <list type="bullet">
/// <item><description>language: ISO-639 language code (mandatory)</description></item>
/// <item><description>country: ISO-3166 country code (optional)</description></item>
@@ -50,13 +38,6 @@ namespace Lucene.Net.Collation
/// </list>
/// </para>
/// <para>
- /// Using a Tailored ruleset:
- /// <list type="bullet">
- /// <item><description>custom: UTF-8 text file containing rules supported by RuleBasedCollator (mandatory)</description></item>
- /// <item><description>strength: 'primary','secondary','tertiary', or 'identical' (optional)</description></item>
- /// <item><description>decomposition: 'no','canonical', or 'full' (optional)</description></item>
- /// </list>
- ///
/// <code>
/// <fieldType name="text_clltnky" class="solr.TextField" positionIncrementGap="100">
/// <analyzer>
@@ -64,18 +45,24 @@ namespace Lucene.Net.Collation
/// <filter class="solr.CollationKeyFilterFactory" language="ja" country="JP"/>
/// </analyzer>
/// </fieldType></code>
- ///
+ /// </para>
+ /// <para>
+ /// <strong>LUCENENET NOTE:</strong> Unlike Lucene's <c>java.text.RuleBasedCollator</c>, the .NET
+ /// <see cref="CompareInfo"/> does not support tailored (custom) collation rules. The <c>custom</c>
+ /// parameter that exists in Lucene is therefore not supported; specifying it throws
+ /// <see cref="NotSupportedException"/>. Use the <c>ICUCollationKeyFilterFactory</c> in the
+ /// Lucene.Net.ICU package if you need custom rules.
/// </para>
/// </summary>
- /// <seealso cref="Collator"/>
+ /// <seealso cref="CompareInfo"/>
/// <seealso cref="CultureInfo"/>
- /// <seealso cref="RuleBasedCollator"/>
/// @since solr 3.1
/// @deprecated use <see cref="CollationKeyAnalyzer"/> instead.
- [Obsolete("use <seealso cref=\"CollationKeyAnalyzer\"/> instead.")]
+ [Obsolete("use CollationKeyAnalyzer instead.")]
public class CollationKeyFilterFactory : TokenFilterFactory, IMultiTermAwareComponent, IResourceLoaderAware
{
- private Collator collator;
+ private CompareInfo collator;
+ private CompareOptions options;
private readonly string custom;
private readonly string language;
private readonly string country;
@@ -99,7 +86,9 @@ public CollationKeyFilterFactory(IDictionary<string, string> args) : base(args)
if (this.custom != null && (this.language != null || this.country != null || this.variant != null))
{
- throw new ArgumentException("Cannot specify both language and custom. " + "To tailor rules for a built-in language, see the javadocs for RuleBasedCollator. " + "Then save the entire customized ruleset to a file, and use with the custom parameter");
+ throw new ArgumentException("Cannot specify both language and custom. "
+ + "To tailor rules for a built-in language, the platform collator must be replaced with the "
+ + "ICUCollationKeyFilterFactory in the Lucene.Net.ICU package, which supports custom rulesets.");
}
if (args.Count > 0)
@@ -112,33 +101,40 @@ public virtual void Inform(IResourceLoader loader)
{
if (this.language != null)
{
- // create from a system collator, based on Locale.
+ // create from a system collator, based on culture.
this.collator = this.CreateFromLocale(this.language, this.country, this.variant);
}
else
{
- // create from a custom ruleset
- this.collator = this.CreateFromRules(this.custom, loader);
+ // LUCENENET: The .NET CompareInfo does not support tailored (custom) collation
+ // rules the way java.text.RuleBasedCollator does. Direct users to the ICU package.
+ throw new NotSupportedException("Custom collation rules are not supported by the platform collator. "
+ + "Use the ICUCollationKeyFilterFactory in the Lucene.Net.ICU package, which supports custom rulesets.");
}
+ this.options = CompareOptions.None;
+
// set the strength flag, otherwise it will be the default.
+ // LUCENENET: System.Globalization.CompareInfo has no Strength concept; the collation
+ // strength is approximated with CompareOptions. IDENTICAL is the strongest level the
+ // platform collator exposes (it has no level beyond TERTIARY/IDENTICAL).
if (this.strength != null)
{
if (this.strength.Equals("primary", StringComparison.OrdinalIgnoreCase))
{
- this.collator.Strength = CollationStrength.Primary;
+ this.options |= CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace;
}
else if (this.strength.Equals("secondary", StringComparison.OrdinalIgnoreCase))
{
- this.collator.Strength = CollationStrength.Secondary;
+ this.options |= CompareOptions.IgnoreCase;
}
else if (this.strength.Equals("tertiary", StringComparison.OrdinalIgnoreCase))
{
- this.collator.Strength = CollationStrength.Tertiary;
+ // default: case- and accent-sensitive (CompareOptions.None)
}
else if (this.strength.Equals("identical", StringComparison.OrdinalIgnoreCase))
{
- this.collator.Strength = CollationStrength.Identical;
+ // default: the platform collator has no level beyond tertiary/identical
}
else
{
@@ -146,22 +142,23 @@ public virtual void Inform(IResourceLoader loader)
}
}
- // LUCENENET TODO: Verify Decomposition > NormalizationMode mapping between the JDK and icu-dotnet
-
// set the decomposition flag, otherwise it will be the default.
+ // LUCENENET: The platform collator always normalizes canonically-equivalent text, so
+ // 'no' and 'canonical' both rely on the default behavior. 'full' additionally folds
+ // compatibility differences such as full-/half-width and kana type.
if (this.decomposition != null)
{
if (this.decomposition.Equals("no", StringComparison.OrdinalIgnoreCase))
{
- this.collator.NormalizationMode = NormalizationMode.Default; // .Decomposition = Collator.NoDecomposition;
+ // default
}
else if (this.decomposition.Equals("canonical", StringComparison.OrdinalIgnoreCase))
{
- this.collator.NormalizationMode = NormalizationMode.Off; //.Decomposition = Collator.CannonicalDecomposition;
+ // default
}
else if (this.decomposition.Equals("full", StringComparison.OrdinalIgnoreCase))
{
- this.collator.NormalizationMode = NormalizationMode.On; //.Decomposition = Collator.FullDecomposition;
+ this.options |= CompareOptions.IgnoreWidth | CompareOptions.IgnoreKanaType;
}
else
{
@@ -172,97 +169,40 @@ public virtual void Inform(IResourceLoader loader)
public override TokenStream Create(TokenStream input)
{
- return new CollationKeyFilter(input, this.collator);
+ return new CollationKeyFilter(input, this.collator, this.options);
}
/// <summary>
- /// Create a locale from language, with optional country and variant.
- /// Then return the appropriate collator for the locale.
+ /// Create a culture from language, with optional country and variant.
+ /// Then return the appropriate collator for the culture.
/// </summary>
- private Collator CreateFromLocale(string language, string country, string variant)
+ private CompareInfo CreateFromLocale(string language, string country, string variant)
{
- CultureInfo cultureInfo;
-
if (language is null)
{
throw new ArgumentException("Language is required");
}
- if (language != null && country is null && variant != null)
+ if (country is null && variant != null)
{
throw new ArgumentException("To specify variant, country is required");
}
+ string name;
if (country != null && variant != null)
{
- cultureInfo = new CultureInfo(string.Concat(language, "-", country, "-", variant));
-
- // LUCENENET TODO: This method won't work on .NET core - confirm the above solution works as expected.
- //cultureInfo = CultureInfo.GetCultures(CultureTypes.SpecificCultures).Single(x =>
- //{
- // if (!x.TwoLetterISOLanguageName.Equals(language, StringComparison.OrdinalIgnoreCase) &&
- // !x.ThreeLetterISOLanguageName.Equals(language, StringComparison.OrdinalIgnoreCase) &&
- // !x.ThreeLetterWindowsLanguageName.Equals(language, StringComparison.OrdinalIgnoreCase))
- // {
- // return false;
- // }
-
- // var region = new RegionInfo(x.Name);
-
- // if (!region.TwoLetterISORegionName.Equals(country, StringComparison.OrdinalIgnoreCase) &&
- // !region.ThreeLetterISORegionName.Equals(country, StringComparison.OrdinalIgnoreCase) &&
- // !region.ThreeLetterWindowsRegionName.Equals(country, StringComparison.OrdinalIgnoreCase)
- // )
- // {
- // return false;
- // }
-
- // return x.Name
- // .Replace(x.TwoLetterISOLanguageName, string.Empty)
- // .Replace(region.TwoLetterISORegionName, string.Empty)
- // .Replace("-", string.Empty)
- // .Equals(variant, StringComparison.OrdinalIgnoreCase);
- //});
+ name = string.Concat(language, "-", country, "-", variant);
}
else if (country != null)
{
- cultureInfo = new CultureInfo(string.Concat(language, "-", country));
+ name = string.Concat(language, "-", country);
}
else
{
- cultureInfo = new CultureInfo(language);
+ name = language;
}
- return Collator.Create(cultureInfo);
- }
-
- /// <summary>
- /// Read custom rules from a file, and create a RuleBasedCollator
- /// The file cannot support comments, as # might be in the rules!
- /// </summary>
- private Collator CreateFromRules(string fileName, IResourceLoader loader)
- {
- Stream input = null;
- try
- {
- input = loader.OpenResource(fileName);
- var rules = ToUTF8String(input);
- return new RuleBasedCollator(rules);
- }
- catch (TransliteratorParseException e)
- {
- // invalid rules
- throw new IOException("ParseException thrown while parsing rules", e);
- }
- catch (SyntaxErrorException e)
- {
- // invalid rules
- throw new IOException("ParseException thrown while parsing rules", e);
- }
- finally
- {
- IOUtils.CloseWhileHandlingException(input);
- }
+ return CompareInfo.GetCompareInfo(name);
}
public virtual AbstractAnalysisFactory GetMultiTermComponent()
@@ -270,21 +210,6 @@ public virtual AbstractAnalysisFactory GetMultiTermComponent()
return this;
}
- private static string ToUTF8String(Stream @in)
- {
- var builder = new StringBuilder();
- var buffer = new char[1024];
- var reader = IOUtils.GetDecodingReader(@in, Encoding.UTF8);
-
- var index = 0;
- while ((index = reader.Read(buffer, index, 1)) > 0)
- {
- builder.Append(buffer, 0, index);
- }
-
- return builder.ToString();
- }
-
/// <summary>
/// Trys to gets the value of a key from a dictionary and removes the value after.
/// This is to mimic java's Dictionary.Remove method.
@@ -302,4 +227,3 @@ private string RemoveFromDictionary(IDictionary<string, string> args, string key
}
}
}
-#endif
diff --git a/src/Lucene.Net.Analysis.Common/Collation/TokenAttributes/CollatedTermAttributeImpl.cs b/src/Lucene.Net.Analysis.Common/Collation/TokenAttributes/CollatedTermAttributeImpl.cs
index 1a1ea17a3..4c058c37e 100644
--- a/src/Lucene.Net.Analysis.Common/Collation/TokenAttributes/CollatedTermAttributeImpl.cs
+++ b/src/Lucene.Net.Analysis.Common/Collation/TokenAttributes/CollatedTermAttributeImpl.cs
@@ -1,8 +1,8 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using Icu.Collation;
using Lucene.Net.Analysis.TokenAttributes;
+using Lucene.Net.Util;
using System;
+using System.Globalization;
namespace Lucene.Net.Collation.TokenAttributes
{
@@ -27,29 +27,62 @@ namespace Lucene.Net.Collation.TokenAttributes
/// Extension of <see cref="CharTermAttribute"/> that encodes the term
/// text as a binary Unicode collation key instead of as UTF-8 bytes.
/// </summary>
- public class CollatedTermAttributeImpl : CharTermAttribute
+ public class CollatedTermAttribute : CharTermAttribute
{
- private readonly Collator collator;
+ private readonly CompareInfo collator;
+ private readonly CompareOptions options;
/// <summary>
- /// Create a new <see cref="CollatedTermAttributeImpl"/> </summary>
+ /// Create a new <see cref="CollatedTermAttribute"/> </summary>
/// <param name="collator"> Collation key generator </param>
- public CollatedTermAttributeImpl(Collator collator)
+ public CollatedTermAttribute(CompareInfo collator)
+ : this(collator, CompareOptions.None)
{
- // clone in case JRE doesn't properly sync,
- // or to reduce contention in case they do
- this.collator = (Collator)collator.Clone();
}
- public override void FillBytesRef()
+ /// <summary>
+ /// Create a new <see cref="CollatedTermAttribute"/> </summary>
+ /// <param name="collator"> Collation key generator </param>
+ /// <param name="options"> Collation options that control the collation strength and
+ /// Unicode normalization (decomposition) of the generated sort key. </param>
+ public CollatedTermAttribute(CompareInfo collator, CompareOptions options)
{
- var bytes = this.BytesRef;
+ // LUCENENET: Unlike java.text.Collator, System.Globalization.CompareInfo is
+ // immutable and thread-safe, so there is no need to clone it here.
+ this.collator = collator;
+ this.options = options;
+ }
- //LUCENENET TODO: Verify that this is correct. Java's byte[] is signed and Big Endian, .NET's is unsigned and Little Endian.
- bytes.Bytes = this.collator.GetSortKey(this.ToString()).KeyData;
+ public override void FillBytesRef()
+ {
+ BytesRef bytes = this.BytesRef;
+#if FEATURE_COMPAREINFO_GETSORTKEY_SPAN
+ // LUCENENET: On .NET 5+ with ICU, CompareInfo has a ReadOnlySpan<char> overload of GetSortKey
+ // that writes directly into a destination buffer, so we can generate the sort key from the term
+ // text without the intermediate string allocation. The NLS backend does not normalize internally
+ // (see Normalize), so we still need the slow path when the app is configured for NLS.
+ if (CollationUtil.IsICU)
+ {
+ ReadOnlySpan<char> source = this.AsSpan();
+ int keyLength = this.collator.GetSortKeyLength(source, this.options);
+ bytes.Bytes = new byte[keyLength];
+ bytes.Offset = 0;
+ bytes.Length = this.collator.GetSortKey(source, bytes.Bytes, this.options);
+ return;
+ }
+#endif
+ byte[] keyData = this.collator.GetSortKey(Normalize(this.ToString()), this.options).KeyData;
+ bytes.Bytes = keyData;
bytes.Offset = 0;
- bytes.Length = bytes.Bytes.Length;
+ bytes.Length = keyData.Length;
}
+
+ // LUCENENET: Normalize the term to Unicode Normalization Form C (NFC) before generating the
+ // sort key. ICU-backed collators (.NET 5+) normalize internally, but the NLS-backed collator
+ // (.NET Framework) does not fully handle decomposed combining sequences (e.g. "I" + U+0307 vs
+ // the precomposed "İ"), so normalizing here keeps sort keys consistent across globalization
+ // backends. This is a no-op for text that is already normalized.
+ internal static string Normalize(string s)
+ => s.IsNormalized() ? s : s.Normalize();
}
}
-#endif
diff --git a/src/Lucene.Net.Analysis.Common/Collation/package.md b/src/Lucene.Net.Analysis.Common/Collation/package.md
index a6c8070e1..229172ea9 100644
--- a/src/Lucene.Net.Analysis.Common/Collation/package.md
+++ b/src/Lucene.Net.Analysis.Common/Collation/package.md
@@ -20,7 +20,7 @@ summary: *content
limitations under the License.
-->
- Unicode collation support. `Collation` converts each token into its binary `CollationKey` using the provided `Collator`, allowing it to be stored as an index term.
+ Unicode collation support. `Collation` converts each token into its binary `System.Globalization.SortKey` using the provided `System.Globalization.CompareInfo` (the .NET platform collator), allowing it to be stored as an index term.
## Use Cases
@@ -40,73 +40,96 @@ summary: *content
### Farsi Range Queries
- // "fa" Locale is not supported by Sun JDK 1.4 or 1.5
- Collator collator = Collator.getInstance(new Locale("ar"));
- CollationKeyAnalyzer analyzer = new CollationKeyAnalyzer(version, collator);
- RAMDirectory ramDir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(version, analyzer));
- Document doc = new Document();
- doc.add(new TextField("content", "\u0633\u0627\u0628", Field.Store.YES));
- writer.addDocument(doc);
- writer.close();
- IndexReader ir = DirectoryReader.open(ramDir);
- IndexSearcher is = new IndexSearcher(ir);
-
- QueryParser aqp = new QueryParser(version, "content", analyzer);
- aqp.setAnalyzeRangeTerms(true);
-
- // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
- // orders the U+0698 character before the U+0633 character, so the single
- // indexed Term above should NOT be returned by a ConstantScoreRangeQuery
- // with a Farsi Collator (or an Arabic one for the case when Farsi is not
- // supported).
- ScoreDoc[] result
- = is.search(aqp.parse("[ \u062F TO \u0698 ]"), null, 1000).scoreDocs;
- assertEquals("The index Term should not be included.", 0, result.length);
+```cs
+CompareInfo collator = CompareInfo.GetCompareInfo("fa");
+CollationKeyAnalyzer analyzer = new CollationKeyAnalyzer(LuceneVersion.LUCENE_48, collator);
+Store.Directory ramDir = new RAMDirectory();
+using (IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer)))
+{
+ Document doc = new Document();
+ doc.Add(new TextField("content", "ساب", Field.Store.YES));
+ writer.AddDocument(doc);
+}
+using IndexReader ir = DirectoryReader.Open(ramDir);
+IndexSearcher searcher = new IndexSearcher(ir);
+
+QueryParser aqp = new QueryParser(LuceneVersion.LUCENE_48, "content", analyzer);
+aqp.AnalyzeRangeTerms = true;
+
+// Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi
+// orders the U+0698 character before the U+0633 character, so the single
+// indexed Term above should NOT be returned by a TermRangeQuery with a
+// Farsi Collator.
+ScoreDoc[] result = searcher.Search(aqp.Parse("[ د TO ژ ]"), null, 1000).ScoreDocs;
+assertEquals("The index Term should not be included.", 0, result.Length);
+```
### Danish Sorting
- Analyzer analyzer
- = new CollationKeyAnalyzer(version, Collator.getInstance(new Locale("da", "dk")));
- RAMDirectory indexStore = new RAMDirectory();
- IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(version, analyzer));
- String[] tracer = new String[] { "A", "B", "C", "D", "E" };
- String[] data = new String[] { "HAT", "HUT", "H\u00C5T", "H\u00D8T", "HOT" };
- String[] sortedTracerOrder = new String[] { "A", "E", "B", "D", "C" };
- for (int i = 0 ; i < data.length="" ;="" ++i)="" {="" document="" doc="new" document();="" doc.add(new="" storedfield("tracer",="" tracer[i]));="" doc.add(new="" textfield("contents",="" data[i],="" field.store.no));="" writer.adddocument(doc);="" }="" writer.close();="" indexreader="" ir="DirectoryReader.open(indexStore);" indexsearcher="" searcher="new" indexsearcher(ir);="" sort="" sort="new" sort();="" sort.setsort(new="" sortfield("contents",="" sortfield.string));="" query="" [...]
+```cs
+Analyzer analyzer = new CollationKeyAnalyzer(LuceneVersion.LUCENE_48, CompareInfo.GetCompareInfo("da-DK"));
+Store.Directory indexStore = new RAMDirectory();
+using (IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer)))
+{
+ string[] tracer = new string[] { "A", "B", "C", "D", "E" };
+ string[] data = new string[] { "HAT", "HUT", "HÅT", "HØT", "HOT" };
+ for (int i = 0; i < data.Length; ++i)
+ {
+ Document doc = new Document();
+ doc.Add(new StoredField("tracer", tracer[i]));
+ doc.Add(new TextField("contents", data[i], Field.Store.NO));
+ writer.AddDocument(doc);
+ }
+}
+string[] sortedTracerOrder = new string[] { "A", "E", "B", "D", "C" };
+using IndexReader ir = DirectoryReader.Open(indexStore);
+IndexSearcher searcher = new IndexSearcher(ir);
+Sort sort = new Sort();
+sort.SetSort(new SortField("contents", SortFieldType.STRING));
+Query query = new MatchAllDocsQuery();
+ScoreDoc[] result = searcher.Search(query, null, 1000, sort).ScoreDocs;
+for (int i = 0; i < result.Length; ++i)
+{
+ Document doc = searcher.Doc(result[i].Doc);
+ assertEquals(sortedTracerOrder[i], doc.GetValues("tracer")[0]);
+}
+```
### Turkish Case Normalization
- Collator collator = Collator.getInstance(new Locale("tr", "TR"));
- collator.setStrength(Collator.PRIMARY);
- Analyzer analyzer = new CollationKeyAnalyzer(version, collator);
- RAMDirectory ramDir = new RAMDirectory();
- IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(version, analyzer));
- Document doc = new Document();
- doc.add(new TextField("contents", "DIGY", Field.Store.NO));
- writer.addDocument(doc);
- writer.close();
- IndexReader ir = DirectoryReader.open(ramDir);
- IndexSearcher is = new IndexSearcher(ir);
- QueryParser parser = new QueryParser(version, "contents", analyzer);
- Query query = parser.parse("d\u0131gy"); // U+0131: dotless i
- ScoreDoc[] result = is.search(query, null, 1000).scoreDocs;
- assertEquals("The index Term should be included.", 1, result.length);
+```cs
+// Primary collation strength is approximated with CompareOptions on the platform collator.
+CompareInfo collator = CompareInfo.GetCompareInfo("tr-TR");
+Analyzer analyzer = new CollationKeyAnalyzer(LuceneVersion.LUCENE_48, collator,
+ CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace);
+Store.Directory ramDir = new RAMDirectory();
+using (IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig(LuceneVersion.LUCENE_48, analyzer)))
+{
+ Document doc = new Document();
+ doc.Add(new TextField("contents", "DIGY", Field.Store.NO));
+ writer.AddDocument(doc);
+}
+using IndexReader ir = DirectoryReader.Open(ramDir);
+IndexSearcher searcher = new IndexSearcher(ir);
+QueryParser parser = new QueryParser(LuceneVersion.LUCENE_48, "contents", analyzer);
+Query query = parser.Parse("dıgy"); // U+0131: dotless i
+ScoreDoc[] result = searcher.Search(query, null, 1000).ScoreDocs;
+assertEquals("The index Term should be included.", 1, result.Length);
+```
## Caveats and Comparisons
- __WARNING:__ Make sure you use exactly the same `Collator` at index and query time -- `CollationKey`s are only comparable when produced by the same `Collator`. Since {@link java.text.RuleBasedCollator}s are not independently versioned, it is unsafe to search against stored `CollationKey`s unless the following are exactly the same (best practice is to store this information with the index and check that they remain the same at query time):
+ __WARNING:__ Make sure you use exactly the same collator (`System.Globalization.CompareInfo` *and* `System.Globalization.CompareOptions`) at index and query time -- `System.Globalization.SortKey`s are only comparable when produced by the same collator. Since the platform collator is not independently versioned, it is unsafe to search against stored `System.Globalization.SortKey`s unless the following are exactly the same (best practice is to store this information with the index and che [...]
-1. JVM vendor
+1. The .NET runtime version, and the active globalization backend. .NET Framework uses Windows NLS, while .NET 5+ uses ICU by default; the two produce different sort keys and orderings. You can detect the active backend at runtime via <xref:Lucene.Net.Collation.CollationUtil.IsICU> (or its inverse, <xref:Lucene.Net.Collation.CollationUtil.IsNLS>) and store that value with the index to verify it matches at query time.
-2. JVM version, including patch version
+2. The language (and country and variant, if specified) of the culture used when obtaining the collator via `System.Globalization.CompareInfo.GetCompareInfo`.
-3. The language (and country and variant, if specified) of the Locale
- used when constructing the collator via
- {@link java.text.Collator#getInstance(java.util.Locale)}.
+3. The `System.Globalization.CompareOptions` used - which approximate the collation strength (for example `IgnoreCase | IgnoreNonSpace` for primary strength) and Unicode normalization (decomposition).
-4. The collation strength used - see {@link java.text.Collator#setStrength(int)}
+ > [!NOTE]
+ > Unlike Lucene's `java.text.RuleBasedCollator`, the .NET `System.Globalization.CompareInfo` does not support tailored (custom) collation rules, so <xref:Lucene.Net.Collation.CollationKeyFilterFactory> does not accept a `custom` ruleset. Before generating a sort key, each term is normalized to Unicode Normalization Form C (NFC) so that decomposed and precomposed input collate consistently across globalization backends.
- `ICUCollationKeyAnalyzer`, available in the [icu analysis module]({@docRoot}/../analyzers-icu/overview-summary.html), uses ICU4J's `Collator`, which makes its version available, thus allowing collation to be versioned independently from the JVM. `ICUCollationKeyAnalyzer` is also significantly faster and generates significantly shorter keys than `CollationKeyAnalyzer`. See [http://site.icu-project.org/charts/collation-icu4j-sun](http://site.icu-project.org/charts/collation-icu4j-sun) for [...]
+ The `ICUCollationKeyAnalyzer`, available in the [Lucene.Net.ICU](../icu/overview.html) package, uses [ICU4N](https://www.nuget.org/packages/ICU4N/)'s `Collator`, which makes its version available, thus allowing collation to be versioned independently from the .NET runtime. `ICUCollationKeyAnalyzer` also generates significantly shorter keys than `CollationKeyAnalyzer`, and supports tailored (custom) rulesets. See [http://site.icu-project.org/charts/collation-icu4j-sun](http://site.icu-pr [...]
- `CollationKey`s generated by `java.text.Collator`s are not compatible with those generated by ICU Collators. Specifically, if you use `CollationKeyAnalyzer` to generate index terms, do not use `ICUCollationKeyAnalyzer` on the query side, or vice versa.
\ No newline at end of file
+ `System.Globalization.SortKey`s generated by the platform collator are not compatible with those generated by ICU Collators. Specifically, if you use `CollationKeyAnalyzer` to generate index terms, do not use `ICUCollationKeyAnalyzer` on the query side, or vice versa.
diff --git a/src/Lucene.Net.Analysis.Common/Support/Collation/CollationUtil.cs b/src/Lucene.Net.Analysis.Common/Support/Collation/CollationUtil.cs
new file mode 100644
index 000000000..ec38ade50
--- /dev/null
+++ b/src/Lucene.Net.Analysis.Common/Support/Collation/CollationUtil.cs
@@ -0,0 +1,58 @@
+using System.Globalization;
+
+namespace Lucene.Net.Collation
+{
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// LUCENENET specific helpers for working with the platform collator
+ /// (<see cref="CompareInfo"/>) used by <see cref="CollationKeyAnalyzer"/> and
+ /// <see cref="CollationAttributeFactory"/>.
+ /// </summary>
+ internal static class CollationUtil
+ {
+ /// <summary>
+ /// Indicates whether the active globalization backend is ICU (the default on .NET 5+)
+ /// rather than the legacy NLS implementation. .NET Framework always uses NLS, while .NET 5+
+ /// uses ICU by default but can be configured to use NLS (for example via the
+ /// <c>System.Globalization.UseNls</c> runtime switch or the <c>DOTNET_SYSTEM_GLOBALIZATION_USENLS</c>
+ /// environment variable). NLS and ICU produce different sort keys and orderings, so this value
+ /// (along with the runtime version and culture) is part of what must match between index time
+ /// and query time when searching against stored sort keys.
+ /// <para/>
+ /// See <a href="https://learn.microsoft.com/en-us/dotnet/core/extensions/globalization-icu#determine-if-your-app-is-using-icu">Determine if your app is using ICU</a>.
+ /// </summary>
+ public static bool IsICU { get; } = DetectICU();
+
+ /// <summary>
+ /// Indicates whether the active globalization backend is the legacy NLS implementation
+ /// (always the case on .NET Framework, and the default on Windows when the app opts out of ICU)
+ /// rather than ICU. This is the inverse of <see cref="IsICU"/>.
+ /// </summary>
+ public static bool IsNLS => !IsICU;
+
+ // LUCENENET: Detects whether globalization is backed by ICU. See the link in the IsICU docs.
+ private static bool DetectICU()
+ {
+ SortVersion sortVersion = CultureInfo.InvariantCulture.CompareInfo.Version;
+ byte[] bytes = sortVersion.SortId.ToByteArray();
+ int version = bytes[3] << 24 | bytes[2] << 16 | bytes[1] << 8 | bytes[0];
+ return version != 0 && version == sortVersion.FullVersion;
+ }
+ }
+}
diff --git a/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs
index a947bece1..f59cd7e32 100644
--- a/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs
+++ b/src/Lucene.Net.Benchmark/ByTask/Tasks/NewCollationAnalyzerTask.cs
@@ -33,11 +33,10 @@ public static Type GetAnalyzerType(this NewCollationAnalyzerTask.Implementation
{
switch (impl)
{
- //case NewCollationAnalyzerTask.Implementation.JDK:
- // return typeof(Lucene.Net.Collation.CollationKeyAnalyzer);
+ case NewCollationAnalyzerTask.Implementation.DotNet:
+ return typeof(Lucene.Net.Collation.CollationKeyAnalyzer);
case NewCollationAnalyzerTask.Implementation.ICU:
- return typeof(Lucene.Net.Collation.ICUCollationKeyAnalyzer);
default:
return typeof(Lucene.Net.Collation.ICUCollationKeyAnalyzer);
}
@@ -47,11 +46,12 @@ public static Type GetCollatorType(this NewCollationAnalyzerTask.Implementation
{
switch (impl)
{
- //case NewCollationAnalyzerTask.Implementation.JDK:
- // return typeof(Icu.Collation.Collator);
+ case NewCollationAnalyzerTask.Implementation.DotNet:
+ // LUCENENET: The .NET equivalent of the JDK's java.text.Collator is the
+ // platform collator, System.Globalization.CompareInfo.
+ return typeof(CompareInfo);
case NewCollationAnalyzerTask.Implementation.ICU:
- return typeof(Collator);
default:
return typeof(Collator);
}
@@ -61,18 +61,24 @@ public static Type GetCollatorType(this NewCollationAnalyzerTask.Implementation
public class NewCollationAnalyzerTask : PerfTask
{
/// <summary>
- /// Different Collation implementations: currently
- /// limited to what is provided in ICU.
+ /// Different Collation implementations: the .NET platform collator
+ /// (<see cref="CompareInfo"/>, the equivalent of the JDK's collator) and ICU.
/// <para/>
/// See <a href="http://site.icu-project.org/charts/collation-icu4j-sun">Comparison of implementations</a>
/// </summary>
public enum Implementation
{
- //JDK, // LUCENENET: Not supported
+ // LUCENENET: This value is named JDK in upstream Lucene (the collator comes from the
+ // Java Development Kit's java.text.Collator). It has been renamed to DotNet here because the
+ // equivalent in .NET is the platform collator (System.Globalization.CompareInfo). The "jdk"
+ // (and legacy "bcl") parameter values are still accepted as aliases.
+
+ /// <summary>The .NET platform collator (<see cref="CompareInfo"/>), equivalent to the JDK's <c>java.text.Collator</c>.</summary>
+ DotNet,
ICU
}
- private Implementation impl = Implementation.ICU; //Implementation.JDK;
+ private Implementation impl = Implementation.DotNet;
public NewCollationAnalyzerTask(PerfRunData runData)
: base(runData)
@@ -81,12 +87,13 @@ public NewCollationAnalyzerTask(PerfRunData runData)
internal static Analyzer CreateAnalyzer(CultureInfo locale, Implementation impl)
{
- // LUCENENET specific - senseless to use reflection here because we only have one
- // collator.
- object collator = Collator.GetInstance(locale);
-
- // LUCENENET TODO: The .NET equivalent to create a collator like the one in the JDK is:
- //CompareInfo.GetCompareInfo(locale.Name);
+ // LUCENENET specific - senseless to use reflection here, so we construct the collator
+ // for the chosen implementation directly. The DotNet implementation (named JDK in upstream
+ // Lucene) maps to the .NET platform collator (System.Globalization.CompareInfo); ICU maps
+ // to ICU4N's Collator.
+ object collator = impl == Implementation.ICU
+ ? (object)Collator.GetInstance(locale)
+ : CompareInfo.GetCompareInfo(locale.Name);
Type clazz = impl.GetAnalyzerType();
return (Analyzer)Activator.CreateInstance(clazz,
@@ -132,8 +139,12 @@ public override void SetParams(string @params)
{
if (value.Equals("icu", StringComparison.OrdinalIgnoreCase))
impl = Implementation.ICU;
- //else if (value.Equals("jdk", StringComparison.OrdinalIgnoreCase))
- // impl = Implementation.JDK;
+ // LUCENENET: "dotnet" maps to what upstream Lucene calls "jdk"; we accept "jdk" (and
+ // the legacy "bcl") as aliases so existing Lucene benchmark .alg files keep working.
+ else if (value.Equals("dotnet", StringComparison.OrdinalIgnoreCase)
+ || value.Equals("jdk", StringComparison.OrdinalIgnoreCase)
+ || value.Equals("bcl", StringComparison.OrdinalIgnoreCase))
+ impl = Implementation.DotNet;
else
throw RuntimeException.Create("Unknown parameter " + param);
}
diff --git a/src/Lucene.Net.TestFramework/Analysis/CollationTestBase.cs b/src/Lucene.Net.TestFramework/Analysis/CollationTestBase.cs
index 34d6898f9..be77364fe 100644
--- a/src/Lucene.Net.TestFramework/Analysis/CollationTestBase.cs
+++ b/src/Lucene.Net.TestFramework/Analysis/CollationTestBase.cs
@@ -43,10 +43,21 @@ public abstract class CollationTestBase : LuceneTestCase
protected string m_secondRangeBeginningOriginal = "\u0633";
protected string m_secondRangeEndOriginal = "\u0638";
- // LUCENENET: The all locales may are not available for collation.
- // LUCENENET: Removed this (only) reference to the ICU library, since it has a lot of data and we don't
- // want to unnecessarily reference it in all test projects.
- //protected readonly string[] availableCollationLocales = RuleBasedCollator.GetAvailableCollationLocales().ToArray();
+ /// <summary>
+ /// LUCENENET: The expected ordering of the French field in <see cref="TestCollationKeySort"/> when using
+ /// the platform collator (<see cref="System.Globalization.CompareInfo"/>) depends on the active
+ /// globalization backend. .NET Framework (NLS) applies traditional French (backward) accent ordering and
+ /// produces "EACGI" - the same result as Lucene's <c>java.text.Collator</c> with <c>Locale.FRANCE</c> -
+ /// whereas .NET 5+ (ICU) uses forward accent ordering and produces "ECAGI". <see cref="System.Globalization.CompareInfo"/>
+ /// has no option to select French reverse-accent ordering, and its sort keys are consistent with its
+ /// <see cref="System.Globalization.CompareInfo.Compare(string, string)"/>, so we detect the active behavior
+ /// here rather than hard-coding a single (backend-specific) value. This is only relevant to tests that use
+ /// the platform collator; the ICU-based collation tests have their own expected values.
+ /// </summary>
+ protected static string FrenchResult =>
+ System.Globalization.CompareInfo.GetCompareInfo("fr").Compare("pêche", "péché") < 0
+ ? "EACGI" // backward (traditional French) accent ordering - NLS backend
+ : "ECAGI"; // forward accent ordering - ICU backend
/// <summary>
/// Convenience method to perform the same function as CollationKeyFilter.
@@ -169,7 +180,19 @@ public virtual void TestFarsiTermRangeQuery(Analyzer analyzer, BytesRef firstBeg
{
// document data:
// the tracer field is used to determine which document was hit
- string[][] sortData = new string[][] { new string[] { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" }, new string[] { "B", "y", "HAT", "HAT", "HAT", "HAT" }, new string[] { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" }, new string[] { "D", "y", "HUT", "HUT", "HUT", "HUT" }, new string[] { "E", "x", "peach", "peach", "peach", "peach" }, new string[] { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" }, new [...]
+ string[][] sortData = new string[][] {
+ // tracer contents US France Sweden (sv_SE) Denmark (da_DK)
+ new string[] { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" },
+ new string[] { "B", "y", "HAT", "HAT", "HAT", "HAT" },
+ new string[] { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" },
+ new string[] { "D", "y", "HUT", "HUT", "HUT", "HUT" },
+ new string[] { "E", "x", "peach", "peach", "peach", "peach" },
+ new string[] { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" },
+ new string[] { "G", "x", "sin", "sin", "sin", "sin" },
+ new string[] { "H", "y", "H\u00D8T", "H\u00D8T", "H\u00D8T", "H\u00D8T" },
+ new string[] { "I", "x", "s\u00EDn", "s\u00EDn", "s\u00EDn", "s\u00EDn" },
+ new string[] { "J", "y", "HOT", "HOT", "HOT", "HOT" },
+ };
FieldType customType = new FieldType();
customType.IsStored = true;
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyAnalyzer.cs b/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyAnalyzer.cs
index 437b77fc5..de67b317f 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyAnalyzer.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyAnalyzer.cs
@@ -1,13 +1,8 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using Icu;
-using Icu.Collation;
using Lucene.Net.Analysis;
using Lucene.Net.Util;
using NUnit.Framework;
-using System;
using System.Globalization;
-using System.Linq;
namespace Lucene.Net.Collation
{
@@ -32,13 +27,21 @@ namespace Lucene.Net.Collation
[TestFixture]
public class TestCollationKeyAnalyzer : CollationTestBase
{
+ private readonly CompareInfo collator = CompareInfo.GetCompareInfo("fa");
+ private readonly Analyzer analyzer;
+
+ private readonly BytesRef firstRangeBeginning;
+ private readonly BytesRef firstRangeEnd;
+ private readonly BytesRef secondRangeBeginning;
+ private readonly BytesRef secondRangeEnd;
+
public TestCollationKeyAnalyzer()
{
this.analyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, this.collator);
- this.firstRangeBeginning = new BytesRef(this.collator.GetSortKey(this.FirstRangeBeginningOriginal).KeyData);
- this.firstRangeEnd = new BytesRef(this.collator.GetSortKey(this.FirstRangeEndOriginal).KeyData);
- this.secondRangeBeginning = new BytesRef(this.collator.GetSortKey(this.SecondRangeBeginningOriginal).KeyData);
- this.secondRangeEnd = new BytesRef(this.collator.GetSortKey(this.SecondRangeEndOriginal).KeyData);
+ this.firstRangeBeginning = new BytesRef(this.collator.GetSortKey(m_firstRangeBeginningOriginal).KeyData);
+ this.firstRangeEnd = new BytesRef(this.collator.GetSortKey(m_firstRangeEndOriginal).KeyData);
+ this.secondRangeBeginning = new BytesRef(this.collator.GetSortKey(m_secondRangeBeginningOriginal).KeyData);
+ this.secondRangeEnd = new BytesRef(this.collator.GetSortKey(m_secondRangeEndOriginal).KeyData);
}
/// <summary>
@@ -46,28 +49,7 @@ public TestCollationKeyAnalyzer()
/// for the inherited root locale: Ø's order isn't specified in Locale.US since
/// its not used in english.
/// </summary>
- private readonly bool oStrokeFirst = Collator.Create("en-US").Compare("Ø", "U") < 0;
-
- /// <summary>
- /// Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
- /// RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
- /// characters properly.
- /// </summary>
- private readonly Collator collator = Collator.Create(new CultureInfo("ar"));
- private Analyzer analyzer;
-
- private BytesRef firstRangeBeginning;
- private BytesRef firstRangeEnd;
- private BytesRef secondRangeBeginning;
- private BytesRef secondRangeEnd;
-
- [Test]
- public virtual void TestInitVars()
- {
- var sortKey = this.collator.GetSortKey(this.FirstRangeBeginningOriginal);
-
- var r = new BytesRef(sortKey.KeyData);
- }
+ private readonly bool oStrokeFirst = CompareInfo.GetCompareInfo("en-US").Compare("Ø", "U") < 0;
[Test]
public virtual void TestFarsiRangeFilterCollating()
@@ -90,72 +72,27 @@ public virtual void TestFarsiTermRangeQuery()
[Test]
public virtual void TestCollationKeySort()
{
- Analyzer usAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, GetCollator("en-US"));
- Collator franceCollator = GetCollator("fr");
- franceCollator.FrenchCollation = FrenchCollation.On;
-
- // `useFallback: true` on both Swedish and Danish collators in
- // case the region specific collator is not found.
- Analyzer franceAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, franceCollator);
- Analyzer swedenAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, GetCollator("sv-SE", "sv"));
- Analyzer denmarkAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, GetCollator("da-DK", "da"));
+ Analyzer usAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("en-US"));
+ Analyzer franceAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("fr"));
+ Analyzer swedenAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("sv-SE"));
+ Analyzer denmarkAnalyzer = new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("da-DK"));
var expectedUSKeySort = this.oStrokeFirst ? "BFJHD" : "BFJDH";
- // The ICU Collator and Sun java.text.Collator implementations differ in their
- // orderings - "BFJDH" is the ordering for java.text.Collator for Locale.US.
this.TestCollationKeySort(usAnalyzer, franceAnalyzer, swedenAnalyzer, denmarkAnalyzer,
- expectedUSKeySort, "EACGI", "BJDFH", "BJDHF");
+ expectedUSKeySort, FrenchResult, "BJDFH", "BJDHF");
}
- // Original Java Code:
- //public void testCollationKeySort() throws Exception {
- // Analyzer usAnalyzer
- // = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.getInstance(Locale.US));
- // Analyzer franceAnalyzer
- // = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.getInstance(Locale.FRANCE));
- // Analyzer swedenAnalyzer
- // = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.getInstance(new Locale("sv", "se")));
- // Analyzer denmarkAnalyzer
- // = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.getInstance(new Locale("da", "dk")));
-
- // // The ICU Collator and Sun java.text.Collator implementations differ in their
- // // orderings - "BFJDH" is the ordering for java.text.Collator for Locale.US.
- // testCollationKeySort
- // (usAnalyzer, franceAnalyzer, swedenAnalyzer, denmarkAnalyzer,
- // oStrokeFirst ? "BFJHD" : "BFJDH", "EACGI", "BJDFH", "BJDHF");
- //}
-
[Test]
public virtual void TestThreadSafe()
{
- var iters = 20 * LuceneTestCase.RANDOM_MULTIPLIER;
+ var iters = 20 * LuceneTestCase.RandomMultiplier;
for (var i = 0; i < iters; i++)
{
- var collator = Collator.Create(new CultureInfo("de"));
- collator.Strength = CollationStrength.Primary;
- this.AssertThreadSafe(new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, collator));
+ CompareInfo collator = CompareInfo.GetCompareInfo("de");
+ this.AssertThreadSafe(new CollationKeyAnalyzer(LuceneTestCase.TEST_VERSION_CURRENT, collator,
+ CompareOptions.IgnoreCase | CompareOptions.IgnoreNonSpace));
}
}
-
- /// <summary>
- /// LUCENENET
- /// Get the first available collator based on the given localeNames.
- /// icu.net may not have all the collation rules.
- /// </summary>
- private Collator GetCollator(params string[] localeNames)
- {
- var firstAvailableLocale = localeNames
- .Select(x => new Locale(x))
- .FirstOrDefault(x => availableCollationLocales.Contains(x.Id));
-
- if (firstAvailableLocale == default)
- throw new ArgumentException($"None of the locales are available: {string.Join(", ", localeNames)}");
-
- Collator collator = Collator.Create(firstAvailableLocale.Id);
-
- return collator;
- }
}
}
-#endif
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilter.cs
index bd56f2e98..462f57468 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilter.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilter.cs
@@ -1,15 +1,11 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using System;
-using System.Globalization;
-using System.IO;
-using System.Linq;
-using Icu;
-using Icu.Collation;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.Core;
using Lucene.Net.Util;
using NUnit.Framework;
+using System;
+using System.Globalization;
+using System.IO;
namespace Lucene.Net.Collation
{
@@ -34,44 +30,38 @@ namespace Lucene.Net.Collation
[Obsolete("remove when CollationKeyFilter is removed.")]
public class TestCollationKeyFilter : CollationTestBase
{
+ private readonly CompareInfo collator = CompareInfo.GetCompareInfo("fa");
+ private readonly Analyzer analyzer;
+
+ private readonly BytesRef firstRangeBeginning;
+ private readonly BytesRef firstRangeEnd;
+ private readonly BytesRef secondRangeBeginning;
+ private readonly BytesRef secondRangeEnd;
+
public TestCollationKeyFilter()
{
- this.analyzer = new TestAnalyzer(this, this.collator);
- this.firstRangeBeginning = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(this.FirstRangeBeginningOriginal).KeyData.ToSByteArray()));
- this.firstRangeEnd = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(this.FirstRangeEndOriginal).KeyData.ToSByteArray()));
- this.secondRangeBeginning = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(this.SecondRangeBeginningOriginal).KeyData.ToSByteArray()));
- this.secondRangeEnd = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(this.SecondRangeEndOriginal).KeyData.ToSByteArray()));
+ this.analyzer = new TestAnalyzer(this.collator);
+ this.firstRangeBeginning = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(m_firstRangeBeginningOriginal).KeyData));
+ this.firstRangeEnd = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(m_firstRangeEndOriginal).KeyData));
+ this.secondRangeBeginning = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(m_secondRangeBeginningOriginal).KeyData));
+ this.secondRangeEnd = new BytesRef(this.EncodeCollationKey(this.collator.GetSortKey(m_secondRangeEndOriginal).KeyData));
}
// the sort order of Ø versus U depends on the version of the rules being used
// for the inherited root locale: Ø's order isn't specified in Locale.US since
// its not used in english.
- internal bool oStrokeFirst = Collator.Create(new CultureInfo("")).Compare("Ø", "U") < 0;
-
- // Neither Java 1.4.2 nor 1.5.0 has Farsi Locale collation available in
- // RuleBasedCollator. However, the Arabic Locale seems to order the Farsi
- // characters properly.
- private readonly Collator collator = Collator.Create(new CultureInfo("ar"));
- private Analyzer analyzer;
-
- private BytesRef firstRangeBeginning;
- private BytesRef firstRangeEnd;
- private BytesRef secondRangeBeginning;
- private BytesRef secondRangeEnd;
+ internal bool oStrokeFirst = CompareInfo.GetCompareInfo("").Compare("Ø", "U") < 0;
public sealed class TestAnalyzer : Analyzer
{
- private readonly TestCollationKeyFilter outerInstance;
-
- internal Collator _collator;
+ internal CompareInfo _collator;
- internal TestAnalyzer(TestCollationKeyFilter outerInstance, Collator collator)
+ internal TestAnalyzer(CompareInfo collator)
{
- this.outerInstance = outerInstance;
this._collator = collator;
}
- protected internal override TokenStreamComponents CreateComponents(String fieldName, TextReader reader)
+ protected internal override TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
{
Tokenizer result = new KeywordTokenizer(reader);
return new TokenStreamComponents(result, new CollationKeyFilter(result, this._collator));
@@ -99,54 +89,13 @@ public virtual void TestFarsiTermRangeQuery()
[Test]
public virtual void TestCollationKeySort()
{
- Analyzer usAnalyzer = new TestAnalyzer(this, GetCollator("en-US"));
- Collator franceCollator = GetCollator("fr");
- franceCollator.FrenchCollation = FrenchCollation.On;
- Analyzer franceAnalyzer = new TestAnalyzer(this, franceCollator);
+ Analyzer usAnalyzer = new TestAnalyzer(CompareInfo.GetCompareInfo("en-US"));
+ Analyzer franceAnalyzer = new TestAnalyzer(CompareInfo.GetCompareInfo("fr"));
+ Analyzer swedenAnalyzer = new TestAnalyzer(CompareInfo.GetCompareInfo("sv-SE"));
+ Analyzer denmarkAnalyzer = new TestAnalyzer(CompareInfo.GetCompareInfo("da-DK"));
- // `useFallback: true` on both Swedish and Danish collators in
- // case the region specific collator is not found.
- Analyzer swedenAnalyzer = new TestAnalyzer(this, GetCollator("sv-SE", "sv"));
- Analyzer denmarkAnalyzer = new TestAnalyzer(this, GetCollator("da-DK", "da"));
-
- // The ICU Collator and Sun java.text.Collator implementations differ in their
- // orderings - "BFJDH" is the ordering for java.text.Collator for Locale.US.
this.TestCollationKeySort(usAnalyzer, franceAnalyzer, swedenAnalyzer, denmarkAnalyzer,
- this.oStrokeFirst ? "BFJHD" : "BFJDH", "EACGI", "BJDFH", "BJDHF");
+ this.oStrokeFirst ? "BFJHD" : "BFJDH", FrenchResult, "BJDFH", "BJDHF");
}
-
- private Collator GetCollator(params string[] localeNames)
- {
- var firstMatchingLocale = localeNames
- .Select(x => new Locale(x))
- .FirstOrDefault(x => availableCollationLocales.Contains(x.Id));
-
- if (firstMatchingLocale == default)
- {
- throw new ArgumentException($"Could not find a collator locale matching any of the following: {string.Join(", ", localeNames)}");
- }
-
- Collator collator = RuleBasedCollator.Create(firstMatchingLocale.Id);
-
- return collator;
- }
-
- // Original Java Code:
- //public void testCollationKeySort() throws Exception {
- // Analyzer usAnalyzer = new TestAnalyzer(Collator.getInstance(Locale.US));
- // Analyzer franceAnalyzer
- // = new TestAnalyzer(Collator.getInstance(Locale.FRANCE));
- // Analyzer swedenAnalyzer
- // = new TestAnalyzer(Collator.getInstance(new Locale("sv", "se")));
- // Analyzer denmarkAnalyzer
- // = new TestAnalyzer(Collator.getInstance(new Locale("da", "dk")));
-
- // // The ICU Collator and Sun java.text.Collator implementations differ in their
- // // orderings - "BFJDH" is the ordering for java.text.Collator for Locale.US.
- // testCollationKeySort
- // (usAnalyzer, franceAnalyzer, swedenAnalyzer, denmarkAnalyzer,
- // oStrokeFirst ? "BFJHD" : "BFJDH", "EACGI", "BJDFH", "BJDHF");
- //}
}
}
-#endif
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilterFactory.cs b/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilterFactory.cs
index 92f3d6fa3..42e8bdcbb 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilterFactory.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Collation/TestCollationKeyFilterFactory.cs
@@ -1,16 +1,9 @@
// Lucene version compatibility level 4.8.1
-#if FEATURE_COLLATION
-using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using Icu;
-using Icu.Collation;
using Lucene.Net.Analysis;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Analysis.Util;
using NUnit.Framework;
-using Assert = Lucene.Net.TestFramework.Assert;
-using JCG = J2N.Collections.Generic;
+using System.IO;
namespace Lucene.Net.Collation
{
@@ -40,6 +33,7 @@ public class TestCollationKeyFilterFactory : BaseTokenStreamFactoryTestCase
/// Instead of using LowerCaseFilter, use a turkish collator with primary strength.
/// Then things will sort and match correctly.
/// </summary>
+ [Test]
public virtual void TestBasicUsage()
{
var turkishUpperCase = "I WİLL USE TURKİSH CASING";
@@ -93,49 +87,11 @@ public virtual void TestSecondaryStrength()
AssertCollatesToSame(tsUpper, tsLower);
}
- /// <summary>
- /// For german, you might want oe to sort and match with o umlaut.
- /// This is not the default, but you can make a customized ruleset to do this.
- ///
- /// The default is DIN 5007-1, this shows how to tailor a collator to get DIN 5007-2 behavior.
- /// http://bugs.sun.com/bugdatabase/view_bug.do?bug_id=4423383
- /// </summary>
- [Test]
- public virtual void TestCustomRules()
- {
- // It is possible not to have the collator rules for a specific
- // country, fallback to the language if that is the case.
- var possiblelocales = new[] { new Locale("de-DE"), new Locale("de") };
- var allRules = RuleBasedCollator.GetAvailableCollationLocales();
- var localeToUse = possiblelocales.FirstOrDefault(locl => allRules.Contains(locl.Id));
-
- Assert.True(localeToUse != default, "Should have found a matching collation locale given the two locales to use.");
-
- const string DIN5007_2_tailorings = "& ae , a\u0308 & AE , A\u0308" + "& oe , o\u0308 & OE , O\u0308" + "& ue , u\u0308 & UE , u\u0308";
- var collationRules = Collator.GetCollationRules(localeToUse.Id);
-
- Assert.IsNotNull(collationRules, $"Rules should have been fetched for {localeToUse.Id}");
-
- string tailoredRules = collationRules + DIN5007_2_tailorings;
-
- RuleBasedCollator tailoredCollator = new RuleBasedCollator(tailoredRules);
-
- // at this point, you would save these tailoredRules to a file,
- // and use the custom parameter.
- string germanUmlaut = "Töne";
- string germanOE = "Toene";
- IDictionary<string, string> args = new JCG.Dictionary<string, string>();
- args["custom"] = "rules.txt";
- args["strength"] = "primary";
-#pragma warning disable 612, 618
- CollationKeyFilterFactory factory = new CollationKeyFilterFactory(args);
-#pragma warning restore 612, 618
- factory.Inform(new StringMockResourceLoader(tailoredRules));
- TokenStream tsUmlaut = factory.Create(new MockTokenizer(new StringReader(germanUmlaut), MockTokenizer.KEYWORD, false));
- TokenStream tsOE = factory.Create(new MockTokenizer(new StringReader(germanOE), MockTokenizer.KEYWORD, false));
-
- AssertCollatesToSame(tsUmlaut, tsOE);
- }
+ // LUCENENET: Upstream's testCustomRules() has been omitted because it relies on
+ // RuleBasedCollator tailoring (custom collation rules), which System.Globalization.CompareInfo
+ // (the platform collator) does not support. CollationKeyFilterFactory therefore does not accept
+ // a "custom" ruleset, so there is no equivalent behavior to test here. See:
+ // https://github.com/apache/lucene/blob/f01152a5909fa6059f4f1d4aeb4e14968ef1d8c2/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilterFactory.java#L108-L142
private static void AssertCollatesToSame(TokenStream stream1, TokenStream stream2)
{
@@ -150,9 +106,8 @@ private static void AssertCollatesToSame(TokenStream stream1, TokenStream stream
assertFalse(stream2.IncrementToken());
stream1.End();
stream2.End();
- stream1.Dispose();
- stream2.Dispose();
+ stream1.Close();
+ stream2.Close();
}
}
}
-#endif
diff --git a/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs b/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs
index e952389b7..edb8e7437 100644
--- a/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs
+++ b/src/Lucene.Net.Tests.Benchmark/ByTask/TestPerfTasksLogic.cs
@@ -1010,8 +1010,7 @@ private string[] getLocaleConfig(string localeParam)
[Test]
public void TestCollator()
{
- // LUCENENET specific - we don't have a JDK version of collator
- // so we are using ICU
+ // ICU implementation
var collatorParam = "impl:icu";
// ROOT locale
@@ -1043,6 +1042,31 @@ public void TestCollator()
benchmark = execBenchmark(getCollatorConfig("nb,NO,NY", collatorParam));
expected = new ICUCollationKeyAnalyzer(TEST_VERSION_CURRENT, Collator.GetInstance(new CultureInfo("nb-NO"/*, "NY"*/)));
assertEqualCollation(expected, benchmark.RunData.Analyzer, "foobar");
+
+ // LUCENENET: DotNet implementation (named "jdk" in upstream Lucene) - maps to the .NET
+ // platform collator (System.Globalization.CompareInfo via CollationKeyAnalyzer). This path
+ // was previously unsupported; it is enabled now that the platform-collator port is complete.
+ var dotNetParam = "impl:dotnet";
+
+ // ROOT locale
+ benchmark = execBenchmark(getCollatorConfig("ROOT", dotNetParam));
+ CollationKeyAnalyzer dotNetExpected = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, CultureInfo.InvariantCulture.CompareInfo);
+ assertEqualCollation(dotNetExpected, benchmark.RunData.Analyzer, "foobar");
+
+ // specify just a language
+ benchmark = execBenchmark(getCollatorConfig("de", dotNetParam));
+ dotNetExpected = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("de"));
+ assertEqualCollation(dotNetExpected, benchmark.RunData.Analyzer, "foobar");
+
+ // specify language + country
+ benchmark = execBenchmark(getCollatorConfig("en,US", dotNetParam));
+ dotNetExpected = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("en-US"));
+ assertEqualCollation(dotNetExpected, benchmark.RunData.Analyzer, "foobar");
+
+ // specify language + country + variant
+ benchmark = execBenchmark(getCollatorConfig("nb,NO,NY", dotNetParam));
+ dotNetExpected = new CollationKeyAnalyzer(TEST_VERSION_CURRENT, CompareInfo.GetCompareInfo("nb-NO"/*, "NY"*/));
+ assertEqualCollation(dotNetExpected, benchmark.RunData.Analyzer, "foobar");
}
private void assertEqualCollation(Analyzer a1, Analyzer a2, string text)