You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2025/05/29 15:05:23 UTC
(lucenenet) branch master updated: ThaiTokenizer + ICUTokenizer: Fixed ICU4N BreakIterator issues to address concurrency problems (Fixes #1035, #1135, and #1159) (#1161)
This is an automated email from the ASF dual-hosted git repository.
nightowl888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
The following commit(s) were added to refs/heads/master by this push:
new 5a9516cfc ThaiTokenizer + ICUTokenizer: Fixed ICU4N BreakIterator issues to address concurrency problems (Fixes #1035, #1135, and #1159) (#1161)
5a9516cfc is described below
commit 5a9516cfc15bf89867c4298a7022e4d8ce344648
Author: Shad Storhaug <sh...@shadstorhaug.com>
AuthorDate: Thu May 29 22:05:16 2025 +0700
ThaiTokenizer + ICUTokenizer: Fixed ICU4N BreakIterator issues to address concurrency problems (Fixes #1035, #1135, and #1159) (#1161)
* Lucene.Net.Analysis.Th.ThaiTokenizer + Lucene.Net.Analysis.Icu.Segmentation.ICUTokenizer: Removed static locks that were used to prevent concurent access to BreakIterator. The concurrency issue with BreakIterator has been addressed in ICU4N 60.1.0-alpha.437. Fixes #1044. Fixes #1135.
* .build/dependencies.props: Bumped ICU4N to 60.1.0-alpha.438.g06cf803beb
* azure-pipelines.yml: Bumped SDK version to 9.0.300 so we copy all of the ICU4N satellite assemblies to the build output.
* .build/runbuild.ps1: Bumped minimum SDK version to 9.0.200 so ICU4N satellite assemblies are copied to the build output.
* BUG: Lucene.Net.Analysis.Th.ThaiWordBreaker::SetText(): Clear the transitions array when resetting the text, since there may be state from a previous use.
* Lucene.Net.Analysis.BaseTokenStreamTestCase::CheckAnalysisConsistency(): AssertTokenStreamContents already consumes the token stream and calls Close(), so removed duplicate Close() call in finally block. See #1159.
* Lucene.Net.Analysis.BaseTokenStreamTestCase::AssertTokenStreamContents(): reset the token stream upon error so Close() doesn't throw an invalid state exception that would obscure our real test error message.
* Lucene.Net.Analysis.BaseTokenStreamTestCase (AssertTokenStreamComponents() + CheckAnalysisConsistency()): When using MockTokenizer, disable the state checks upon exception so our call to Close() won't throw another exception that may obscure the original test failure.
* Lucene.Net.Analysis.Th.TestThaiAnalyzer::AssertAnalyzer(): When using MockTokenizer, disable the state checks upon exception so our call to Close() won't throw another exception that may obscure the original test failure.
* Lucene.Net.Analysis (BaseTokenStreamTestCase + Th.TestThaiAnalyzer): Log any secondary exceptions from TokenStream.Close(), since we don't want them to obscure our original test failure message. Throw only in the case where there was no other failure and Close() throws.
* .build/dependencies.props: bumped ICU4N to 60.1.0-alpha.438
* Lucene.Net.Analysis.Th.ThaiTokenizer: Removed extra helper methods/overloads that only existed to synchronize access to the BreakIterator instances
---
.build/dependencies.props | 2 +-
.build/runbuild.ps1 | 2 +-
azure-pipelines.yml | 2 +-
.../Analysis/Th/ThaiTokenizer.cs | 143 +++----------
.../Analysis/Icu/Segmentation/ICUTokenizer.cs | 46 +----
.../Analysis/BaseTokenStreamTestCase.cs | 230 ++++++++++++++-------
.../Analysis/Th/TestThaiAnalyzer.cs | 129 ++++--------
7 files changed, 233 insertions(+), 321 deletions(-)
diff --git a/.build/dependencies.props b/.build/dependencies.props
index e4d4215da..f3c8dda3f 100644
--- a/.build/dependencies.props
+++ b/.build/dependencies.props
@@ -33,7 +33,7 @@
https://github.com/apache/lucene-solr/tree/31d7ec7bbfdcd2c4cc61d9d35e962165410b65fe/lucene/analysis/icu/src/data/utr30
Just make sure they are adjusted to the right version of ICU/Lucene.
<ICU4NPackageVersion>[60.1,60.2)</ICU4NPackageVersion> -->
- <ICU4NPackageVersion>[60.1.0-alpha.436,60.1.0-alpha.446)</ICU4NPackageVersion>
+ <ICU4NPackageVersion>[60.1.0-alpha.438,60.1.0-alpha.446)</ICU4NPackageVersion>
<IKVMPackageVersion>8.7.5</IKVMPackageVersion>
<IKVMMavenSdkPackageVersion>1.6.7</IKVMMavenSdkPackageVersion>
<!-- J2N will break binary compatibility in 3.0.0 to fix the APIs of collection types -->
diff --git a/.build/runbuild.ps1 b/.build/runbuild.ps1
index faa26d584..8aa84c2d9 100644
--- a/.build/runbuild.ps1
+++ b/.build/runbuild.ps1
@@ -27,7 +27,7 @@ properties {
[string]$testResultsDirectory = "$artifactsDirectory/TestResults"
[string]$publishDirectory = "$artifactsDirectory/Publish"
[string]$solutionFile = "$baseDirectory/Lucene.Net.sln"
- [string]$minimumSdkVersion = "9.0.100"
+ [string]$minimumSdkVersion = "9.0.200" # We need at least 9.0.200 for ICU4N satellite assemblies with 3-character language codes to get copied to the output.
[string]$globalJsonFile = "$baseDirectory/global.json"
[string]$versionPropsFile = "$baseDirectory/version.props"
[string]$luceneReadmeFile = "$baseDirectory/src/Lucene.Net/readme-nuget.md"
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index 2db143413..7f4089e94 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -57,7 +57,7 @@ variables:
- name: BuildCounter
value: $[counter(variables['VersionSuffix'],coalesce(variables['BuildCounterSeed'], 1250))]
- name: DotNetSDKVersion
- value: '9.0.100'
+ value: '9.0.300'
- name: DocumentationArtifactName
value: 'docs'
- name: DocumentationArtifactZipFileName
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Th/ThaiTokenizer.cs b/src/Lucene.Net.Analysis.Common/Analysis/Th/ThaiTokenizer.cs
index da385b9a6..e79c0f515 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Th/ThaiTokenizer.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Th/ThaiTokenizer.cs
@@ -46,41 +46,13 @@ namespace Lucene.Net.Analysis.Th
/// </summary>
public class ThaiTokenizer : SegmentingTokenizerBase
{
- private static readonly object syncLock = new object(); // LUCENENET specific - workaround until BreakIterator is made thread safe (LUCENENET TODO: TO REVERT)
-
// LUCENENET specific - DBBI_AVAILABLE removed because ICU always has a dictionary-based BreakIterator
- private static readonly BreakIterator proto = LoadProto();
+ private static readonly BreakIterator proto = BreakIterator.GetWordInstance(new CultureInfo("th"));
/// <summary>
/// used for breaking the text into sentences
/// </summary>
- private static readonly BreakIterator sentenceProto = LoadSentenceProto();
-
- private static BreakIterator LoadProto()
- {
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- return BreakIterator.GetWordInstance(new CultureInfo("th"));
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
- }
-
- private static BreakIterator LoadSentenceProto()
- {
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- return BreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
- }
+ private static readonly BreakIterator sentenceProto = BreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
private readonly ThaiWordBreaker wordBreaker;
private readonly CharArrayIterator wrapper = CharArrayIterator.NewWordInstance();
@@ -101,112 +73,48 @@ public ThaiTokenizer(TextReader reader)
/// <summary>
/// Creates a new <see cref="ThaiTokenizer"/>, supplying the <see cref="AttributeFactory"/> </summary>
public ThaiTokenizer(AttributeFactory factory, TextReader reader)
- : base(factory, reader, CreateSentenceClone())
+ : base(factory, reader, (BreakIterator)sentenceProto.Clone())
{
// LUCENENET specific - DBBI_AVAILABLE removed because ICU always has a dictionary-based BreakIterator
-
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- wordBreaker = new ThaiWordBreaker((BreakIterator)proto.Clone());
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
+ wordBreaker = new ThaiWordBreaker((BreakIterator)proto.Clone());
termAtt = AddAttribute<ICharTermAttribute>();
offsetAtt = AddAttribute<IOffsetAttribute>();
}
- private static BreakIterator CreateSentenceClone()
- {
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- return (BreakIterator)sentenceProto.Clone();
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
- }
-
- public override void Reset()
- {
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- base.Reset();
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
- }
-
- public override State CaptureState()
- {
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- return base.CaptureState();
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
- }
-
protected override void SetNextSentence(int sentenceStart, int sentenceEnd)
{
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- this.sentenceStart = sentenceStart;
- this.sentenceEnd = sentenceEnd;
- wrapper.SetText(m_buffer, sentenceStart, sentenceEnd - sentenceStart);
- wordBreaker.SetText(wrapper);
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
+ this.sentenceStart = sentenceStart;
+ this.sentenceEnd = sentenceEnd;
+ wrapper.SetText(m_buffer, sentenceStart, sentenceEnd - sentenceStart);
+ wordBreaker.SetText(wrapper);
}
protected override bool IncrementWord()
{
int start, end;
- UninterruptableMonitor.Enter(syncLock);
- try
+ start = wordBreaker.Current;
+ if (start == BreakIterator.Done)
{
- start = wordBreaker.Current;
- if (start == BreakIterator.Done)
- {
- return false; // BreakIterator exhausted
- }
+ return false; // BreakIterator exhausted
+ }
- // find the next set of boundaries, skipping over non-tokens
+ // find the next set of boundaries, skipping over non-tokens
+ end = wordBreaker.Next();
+ while (end != BreakIterator.Done && !Character.IsLetterOrDigit(Character.CodePointAt(m_buffer, sentenceStart + start, sentenceEnd)))
+ {
+ start = end;
end = wordBreaker.Next();
- while (end != BreakIterator.Done && !Character.IsLetterOrDigit(Character.CodePointAt(m_buffer, sentenceStart + start, sentenceEnd)))
- {
- start = end;
- end = wordBreaker.Next();
- }
-
- if (end == BreakIterator.Done)
- {
- return false; // BreakIterator exhausted
- }
-
- ClearAttributes();
- termAtt.CopyBuffer(m_buffer, sentenceStart + start, end - start);
- offsetAtt.SetOffset(CorrectOffset(m_offset + sentenceStart + start), CorrectOffset(m_offset + sentenceStart + end));
- return true;
}
- finally
+
+ if (end == BreakIterator.Done)
{
- UninterruptableMonitor.Exit(syncLock);
+ return false; // BreakIterator exhausted
}
+
+ ClearAttributes();
+ termAtt.CopyBuffer(m_buffer, sentenceStart + start, end - start);
+ offsetAtt.SetOffset(CorrectOffset(m_offset + sentenceStart + start), CorrectOffset(m_offset + sentenceStart + end));
+ return true;
}
}
@@ -231,6 +139,7 @@ public void SetText(CharArrayIterator text)
{
this.text.CopyChars(text.Text, text.Start, text.Length);
wordBreaker.SetText(text);
+ transitions.Clear();
}
public int Current
diff --git a/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs b/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs
index dec38b200..c552d7f9c 100644
--- a/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs
+++ b/src/Lucene.Net.Analysis.ICU/Analysis/Icu/Segmentation/ICUTokenizer.cs
@@ -58,8 +58,6 @@ public sealed class ICUTokenizer : Tokenizer
private readonly ITypeAttribute typeAtt;
private readonly IScriptAttribute scriptAtt;
- private static readonly object syncLock = new object(); // LUCENENET specific - workaround until BreakIterator is made thread safe (LUCENENET TODO: TO REVERT)
-
/// <summary>
/// Construct a new <see cref="ICUTokenizer"/> that breaks text into words from the given
/// <see cref="TextReader"/>.
@@ -112,39 +110,23 @@ public ICUTokenizer(AttributeFactory factory, TextReader input, ICUTokenizerConf
public override bool IncrementToken()
{
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- ClearAttributes();
- if (length == 0)
- Refill();
- while (!IncrementTokenBuffer())
- {
- Refill();
- if (length <= 0) // no more bytes to read;
- return false;
- }
- return true;
- }
- finally
+ ClearAttributes();
+ if (length == 0)
+ Refill();
+ while (!IncrementTokenBuffer())
{
- UninterruptableMonitor.Exit(syncLock);
+ Refill();
+ if (length <= 0) // no more bytes to read;
+ return false;
}
+ return true;
}
public override void Reset()
{
base.Reset();
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- breaker.SetText(buffer, 0, 0);
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
+ breaker.SetText(buffer, 0, 0);
length = usableLength = offset = 0;
}
@@ -206,15 +188,7 @@ private void Refill()
*/
}
- UninterruptableMonitor.Enter(syncLock);
- try
- {
- breaker.SetText(buffer, 0, Math.Max(0, usableLength));
- }
- finally
- {
- UninterruptableMonitor.Exit(syncLock);
- }
+ breaker.SetText(buffer, 0, Math.Max(0, usableLength));
}
// TODO: refactor to a shared readFully somewhere
diff --git a/src/Lucene.Net.TestFramework/Analysis/BaseTokenStreamTestCase.cs b/src/Lucene.Net.TestFramework/Analysis/BaseTokenStreamTestCase.cs
index bc7783112..fdeac4e9f 100644
--- a/src/Lucene.Net.TestFramework/Analysis/BaseTokenStreamTestCase.cs
+++ b/src/Lucene.Net.TestFramework/Analysis/BaseTokenStreamTestCase.cs
@@ -117,8 +117,11 @@ public abstract class BaseTokenStreamTestCase : LuceneTestCase
public static void AssertTokenStreamContents(TokenStream ts, string[] output, int[] startOffsets, int[] endOffsets, string[] types, int[] posIncrements, int[] posLengths, int? finalOffset, int? finalPosInc, bool[] keywordAtts, bool offsetsAreCorrect, byte[][] payloads)
{
// LUCENENET: Bug fix: NUnit throws an exception when something fails.
- // This causes Dispose() to be skipped and it pollutes other tests indicating false negatives.
- // Added this try-finally block to fix this.
+ // This causes Close() to be skipped and it pollutes other tests indicating false negatives
+ // if there are open file handles. We added this try-finally block to fix this, but we must
+ // take care not to swallow our original exception message, so we track whether or not the
+ // try block completed and only throw an exception if it did, otherwise log the error (Verbose ignored).
+ bool success = false;
try
{
Assert.IsNotNull(output);
@@ -381,10 +384,25 @@ public static void AssertTokenStreamContents(TokenStream ts, string[] output, in
{
Assert.AreEqual((int)finalPosInc, posIncrAtt.PositionIncrement, "finalPosInc");
}
+
+ // LUCENENET: Keep track of successful completion for the finally block
+ success = true;
}
finally
{
- ts.Close();
+ // LUCENENET: We are doing this in the finally block to ensure it happens
+ // when there are exceptions thrown (such as when the assert fails).
+ try
+ {
+ ts.Close();
+ }
+ // LUCENENET: Make the failure as visible as possible, but do not rethrow it
+ // in case of a primary exception because it may swallow the information required
+ // to reproduce the failure.
+ catch (Exception e) when (!success)
+ {
+ Console.WriteLine($"BaseTokenStreamTestCase.AssertTokenStreamContents(): Failed to close TokenStream: {e}");
+ }
}
}
@@ -934,6 +952,14 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
int remainder = random.Next(10);
TextReader reader = new StringReader(text);
TokenStream ts = a.GetTokenStream("dummy", useCharFilter ? new MockCharFilter(reader, remainder) : reader);
+
+ // LUCENENET: Bug fix: NUnit throws an exception when something fails.
+ // This causes Close() to be skipped and it pollutes other tests indicating false negatives
+ // if there are open file handles. We added this try-finally block to fix this, but we must
+ // take care not to swallow our original exception message, so we track whether or not the
+ // try block completed and only throw an exception if it did, otherwise log the error (Verbose ignored).
+ bool success = false;
+
ICharTermAttribute termAtt = ts.HasAttribute<ICharTermAttribute>() ? ts.GetAttribute<ICharTermAttribute>() : null;
IOffsetAttribute offsetAtt = ts.HasAttribute<IOffsetAttribute>() ? ts.GetAttribute<IOffsetAttribute>() : null;
IPositionIncrementAttribute posIncAtt = ts.HasAttribute<IPositionIncrementAttribute>() ? ts.GetAttribute<IPositionIncrementAttribute>() : null;
@@ -977,12 +1003,25 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
}
ts.End();
+
+ // LUCENENET: Keep track of successful completion for the finally block
+ success = true;
}
finally
{
// LUCENENET: We are doing this in the finally block to ensure it happens
// when there are exceptions thrown (such as when the assert fails).
- ts.Close();
+ try
+ {
+ ts.Close();
+ }
+ // LUCENENET: Make the failure as visible as possible, but do not rethrow it
+ // in case of a primary exception because it may swallow the information required
+ // to reproduce the failure.
+ catch (Exception e) when (!success)
+ {
+ Console.WriteLine($"BaseTokenStreamTestCase.CheckAnalysisConsistency() - (1): Failed to close TokenStream: {e}");
+ }
}
// verify reusing is "reproducable" and also get the normal tokenstream sanity checks
@@ -1027,9 +1066,19 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
Assert.IsTrue(MockReaderWrapper.IsMyEvilException(re));
}
+ // LUCENENET: Bug fix: NUnit throws an exception when something fails.
+ // This causes Close() to be skipped and it pollutes other tests indicating false negatives
+ // if there are open file handles. We added this try-finally block to fix this, but we must
+ // take care not to swallow our original exception message, so we track whether or not the
+ // try block completed and only throw an exception if it did, otherwise log the error (Verbose ignored).
+ success = false;
+
try
{
ts.End();
+
+ // LUCENENET: Keep track of successful completion for the finally block
+ success = true;
}
catch (Exception ae) when (ae.IsAssertionError() && ae.Message.Contains("End() called before IncrementToken() returned false!"))
{
@@ -1039,7 +1088,17 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
}
finally
{
- ts.Close();
+ try
+ {
+ ts.Close();
+ }
+ // LUCENENET: Make the failure as visible as possible, but do not rethrow it
+ // in case of a primary exception because it may swallow the information required
+ // to reproduce the failure.
+ catch (Exception e) when (!success)
+ {
+ Console.WriteLine($"BaseTokenStreamTestCase.CheckAnalysisConsistency() - (2): Failed to close TokenStream: {e}");
+ }
}
}
else if (evilness == 7)
@@ -1059,9 +1118,19 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
Assert.IsTrue(ts.IncrementToken());
}
+ // LUCENENET: Bug fix: NUnit throws an exception when something fails.
+ // This causes Close() to be skipped and it pollutes other tests indicating false negatives
+ // if there are open file handles. We added this try-finally block to fix this, but we must
+ // take care not to swallow our original exception message, so we track whether or not the
+ // try block completed and only throw an exception if it did, otherwise log the error (Verbose ignored).
+ success = false;
+
try
{
ts.End();
+
+ // LUCENENET: Keep track of successful completion for the finally block
+ success = true;
}
catch (Exception ae) when (ae.IsAssertionError() && ae.Message.Contains("End() called before IncrementToken() returned false!"))
{
@@ -1071,7 +1140,17 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
}
finally
{
- ts.Close();
+ try
+ {
+ ts.Close();
+ }
+ // LUCENENET: Make the failure as visible as possible, but do not rethrow it
+ // in case of a primary exception because it may swallow the information required
+ // to reproduce the failure.
+ catch (Exception e) when (!success)
+ {
+ Console.WriteLine($"BaseTokenStreamTestCase.CheckAnalysisConsistency() - (3): Failed to close TokenStream: {e}");
+ }
}
}
}
@@ -1100,82 +1179,75 @@ private static void CheckAnalysisConsistency(Random random, Analyzer a, bool use
ts = a.GetTokenStream("dummy", useCharFilter ? new MockCharFilter(reader, remainder) : reader);
- try
+ if (typeAtt != null && posIncAtt != null && posLengthAtt != null && offsetAtt != null)
{
- if (typeAtt != null && posIncAtt != null && posLengthAtt != null && offsetAtt != null)
- {
- // offset + pos + posLength + type
- AssertTokenStreamContents(ts,
- tokens.ToArray(),
- ToIntArray(startOffsets),
- ToIntArray(endOffsets),
- types.ToArray(),
- ToIntArray(positions),
- ToIntArray(positionLengths),
- text.Length,
- offsetsAreCorrect);
- }
- else if (typeAtt != null && posIncAtt != null && offsetAtt != null)
- {
- // offset + pos + type
- AssertTokenStreamContents(ts,
- tokens.ToArray(),
- ToIntArray(startOffsets),
- ToIntArray(endOffsets),
- types.ToArray(),
- ToIntArray(positions),
- null,
- text.Length,
- offsetsAreCorrect);
- }
- else if (posIncAtt != null && posLengthAtt != null && offsetAtt != null)
- {
- // offset + pos + posLength
- AssertTokenStreamContents(ts,
- tokens.ToArray(),
- ToIntArray(startOffsets),
- ToIntArray(endOffsets),
- null,
- ToIntArray(positions),
- ToIntArray(positionLengths),
- text.Length,
- offsetsAreCorrect);
- }
- else if (posIncAtt != null && offsetAtt != null)
- {
- // offset + pos
- AssertTokenStreamContents(ts,
- tokens.ToArray(),
- ToIntArray(startOffsets),
- ToIntArray(endOffsets),
- null,
- ToIntArray(positions),
- null,
- text.Length,
- offsetsAreCorrect);
- }
- else if (offsetAtt != null)
- {
- // offset
- AssertTokenStreamContents(ts,
- tokens.ToArray(),
- ToIntArray(startOffsets),
- ToIntArray(endOffsets),
- null,
- null,
- null,
- text.Length,
- offsetsAreCorrect);
- }
- else
- {
- // terms only
- AssertTokenStreamContents(ts, tokens.ToArray());
- }
+ // offset + pos + posLength + type
+ AssertTokenStreamContents(ts,
+ tokens.ToArray(),
+ ToIntArray(startOffsets),
+ ToIntArray(endOffsets),
+ types.ToArray(),
+ ToIntArray(positions),
+ ToIntArray(positionLengths),
+ text.Length,
+ offsetsAreCorrect);
}
- finally
+ else if (typeAtt != null && posIncAtt != null && offsetAtt != null)
{
- ts.Close();
+ // offset + pos + type
+ AssertTokenStreamContents(ts,
+ tokens.ToArray(),
+ ToIntArray(startOffsets),
+ ToIntArray(endOffsets),
+ types.ToArray(),
+ ToIntArray(positions),
+ null,
+ text.Length,
+ offsetsAreCorrect);
+ }
+ else if (posIncAtt != null && posLengthAtt != null && offsetAtt != null)
+ {
+ // offset + pos + posLength
+ AssertTokenStreamContents(ts,
+ tokens.ToArray(),
+ ToIntArray(startOffsets),
+ ToIntArray(endOffsets),
+ null,
+ ToIntArray(positions),
+ ToIntArray(positionLengths),
+ text.Length,
+ offsetsAreCorrect);
+ }
+ else if (posIncAtt != null && offsetAtt != null)
+ {
+ // offset + pos
+ AssertTokenStreamContents(ts,
+ tokens.ToArray(),
+ ToIntArray(startOffsets),
+ ToIntArray(endOffsets),
+ null,
+ ToIntArray(positions),
+ null,
+ text.Length,
+ offsetsAreCorrect);
+ }
+ else if (offsetAtt != null)
+ {
+ // offset
+ AssertTokenStreamContents(ts,
+ tokens.ToArray(),
+ ToIntArray(startOffsets),
+ ToIntArray(endOffsets),
+ null,
+ null,
+ null,
+ text.Length,
+ offsetsAreCorrect);
+ }
+ else
+ {
+ // terms only
+ AssertTokenStreamContents(ts, tokens.ToArray());
}
if (field != null)
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Th/TestThaiAnalyzer.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Th/TestThaiAnalyzer.cs
index 50acb5283..5edc58f2c 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Th/TestThaiAnalyzer.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Th/TestThaiAnalyzer.cs
@@ -262,111 +262,69 @@ private static void AssertAnalyzer(Analyzer analyzer, string text)
TextReader reader = new StringReader(text);
TokenStream ts = analyzer.GetTokenStream("dummy", reader);
+ // LUCENENET: Bug fix: NUnit throws an exception when something fails.
+ // This causes Close() to be skipped and it pollutes other tests indicating false negatives
+ // if there are open file handles. We added this try-finally block to fix this, but we must
+ // take care not to swallow our original exception message, so we track whether or not the
+ // try block completed and only throw an exception if it did, otherwise log the error (Verbose ignored).
+ bool success = false;
+
try
{
- bool isReset = false;
- try
- {
+ termAtt = ts.GetAttribute<ICharTermAttribute>();
+ offsetAtt = ts.GetAttribute<IOffsetAttribute>();
+ posIncAtt = ts.GetAttribute<IPositionIncrementAttribute>();
- termAtt = ts.GetAttribute<ICharTermAttribute>();
- offsetAtt = ts.GetAttribute<IOffsetAttribute>();
- posIncAtt = ts.GetAttribute<IPositionIncrementAttribute>();
-
- ts.Reset();
- isReset = true;
-
- while (ts.IncrementToken())
- {
- Assert.IsNotNull(termAtt, "has no CharTermAttribute");
- tokens.Add(termAtt.ToString());
- positions.Add(posIncAtt.PositionIncrement);
- startOffsets.Add(offsetAtt.StartOffset);
- endOffsets.Add(offsetAtt.EndOffset);
- }
- }
- finally
+ ts.Reset();
+
+ while (ts.IncrementToken())
{
- if (!isReset)
- {
- try
- {
- // consume correctly
- ts.Reset();
- while (ts.IncrementToken()) ;
- //ts.End();
- //ts.Dispose();
- }
-#pragma warning disable 168
- catch (Exception ex)
-#pragma warning restore 168
- {
- // ignore
- }
- }
-
- ts.End();
+ Assert.IsNotNull(termAtt, "has no CharTermAttribute");
+ tokens.Add(termAtt.ToString());
+ positions.Add(posIncAtt.PositionIncrement);
+ startOffsets.Add(offsetAtt.StartOffset);
+ endOffsets.Add(offsetAtt.EndOffset);
}
- }
- finally
- {
- ts.Close();
- }
- reader = new StringReader(text);
- ts = analyzer.GetTokenStream("dummy", reader);
+ ts.End();
- try
+ // LUCENENET: Keep track of successful completion for the finally block
+ success = true;
+ }
+ finally
{
- bool isReset = false;
try
{
-
- // offset + pos
- AssertTokenStreamContents(ts,
- output: tokens.ToArray(),
- startOffsets: ToIntArray(startOffsets),
- endOffsets: ToIntArray(endOffsets),
- types: null,
- posIncrements: ToIntArray(positions),
- posLengths: null,
- finalOffset: text.Length,
- offsetsAreCorrect: true);
-
- isReset = true;
+ ts.Close();
}
- finally
+ // LUCENENET: Make the failure as visible as possible, but do not rethrow it
+ // in case of a primary exception because it may swallow the information required
+ // to reproduce the failure.
+ catch (Exception e) when (!success)
{
- if (!isReset)
- {
- try
- {
- // consume correctly
- ts.Reset();
- while (ts.IncrementToken()) ;
- //ts.End();
- //ts.Dispose();
- }
-#pragma warning disable 168
- catch (Exception ex)
-#pragma warning restore 168
- {
- // ignore
- }
- }
- ts.End();
+ Console.WriteLine($"TestThaiAnalyzer.AssertAnalyzer(): Failed to close TokenStream: {e}");
}
}
- finally
- {
- ts.Close();
- }
+
+ reader = new StringReader(text);
+ ts = analyzer.GetTokenStream("dummy", reader);
+
+ // offset + pos
+ AssertTokenStreamContents(ts,
+ output: tokens.ToArray(),
+ startOffsets: ToIntArray(startOffsets),
+ endOffsets: ToIntArray(endOffsets),
+ types: null,
+ posIncrements: ToIntArray(positions),
+ posLengths: null,
+ finalOffset: text.Length,
+ offsetsAreCorrect: true);
}
/// <summary>
/// blast some random strings through the analyzer </summary>
[Test]
- [AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/269")] // LUCENENET TODO: this test occasionally fails
public virtual void TestRandomStrings()
{
CheckRandomData(Random, new ThaiAnalyzer(TEST_VERSION_CURRENT), 1000 * RandomMultiplier);
@@ -376,7 +334,6 @@ public virtual void TestRandomStrings()
/// blast some random large strings through the analyzer </summary>
///
[Test]
- [AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/269")] // LUCENENET TODO: this test occasionally fails
public virtual void TestRandomHugeStrings()
{
Random random = Random;