You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2026/06/28 17:18:33 UTC
(lucenenet) branch master updated: Fix random TestRandomChains failures by excluding broken-offsets producers, #1072 (#1348)
This is an automated email from the ASF dual-hosted git repository.
NightOwl888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
The following commit(s) were added to refs/heads/master by this push:
new 2cde8b5e9 Fix random TestRandomChains failures by excluding broken-offsets producers, #1072 (#1348)
2cde8b5e9 is described below
commit 2cde8b5e9f84ec5f8f4428cd979693093c45c74f
Author: Paul Irwin <pa...@gmail.com>
AuthorDate: Sun Jun 28 11:18:27 2026 -0600
Fix random TestRandomChains failures by excluding broken-offsets producers, #1072 (#1348)
Co-authored-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
.../Compound/HyphenationCompoundWordTokenFilter.cs | 6 +++-
.../JapaneseKatakanaStemFilter.cs | 8 +++--
.../Compound/TestCompoundWordTokenFilter.cs | 36 ++++++++++++++++++++++
.../Analysis/Core/TestRandomChains.cs | 16 +++++++---
.../TestJapaneseKatakanaStemFilter.cs | 21 +++++++++++++
5 files changed, 79 insertions(+), 8 deletions(-)
diff --git a/src/Lucene.Net.Analysis.Common/Analysis/Compound/HyphenationCompoundWordTokenFilter.cs b/src/Lucene.Net.Analysis.Common/Analysis/Compound/HyphenationCompoundWordTokenFilter.cs
index 6ca2f4274..8de21ba87 100644
--- a/src/Lucene.Net.Analysis.Common/Analysis/Compound/HyphenationCompoundWordTokenFilter.cs
+++ b/src/Lucene.Net.Analysis.Common/Analysis/Compound/HyphenationCompoundWordTokenFilter.cs
@@ -244,7 +244,11 @@ protected override void Decompose()
m_tokens.Enqueue(new CompoundToken(this, start, partLength));
}
}
- else if (m_dictionary.Contains(m_termAtt.Buffer, start, partLength - 1))
+ // LUCENENET specific - guard against a negative length (partLength == 0).
+ // Upstream Java relies on CharArrayMap silently treating a negative length as
+ // "not found", but Lucene.NET's CharArrayMap validates its arguments and throws
+ // ArgumentOutOfRangeException. See the BOGUS/BROKEN/FUNKY/WACKO note above.
+ else if (partLength - 1 >= 0 && m_dictionary.Contains(m_termAtt.Buffer, start, partLength - 1))
{
// check the dictionary again with a word that is one character
// shorter
diff --git a/src/Lucene.Net.Analysis.Kuromoji/JapaneseKatakanaStemFilter.cs b/src/Lucene.Net.Analysis.Kuromoji/JapaneseKatakanaStemFilter.cs
index f2b802e40..86f410e5f 100644
--- a/src/Lucene.Net.Analysis.Kuromoji/JapaneseKatakanaStemFilter.cs
+++ b/src/Lucene.Net.Analysis.Kuromoji/JapaneseKatakanaStemFilter.cs
@@ -51,9 +51,11 @@ public sealed class JapaneseKatakanaStemFilter : TokenFilter
public JapaneseKatakanaStemFilter(TokenStream input, int minimumLength)
: base(input)
{
- // LUCENENET: Added guard clause
- if (minimumLength < 0)
- throw new ArgumentOutOfRangeException(nameof(minimumLength), "Minimum length must be a non-negative integer.");
+ // LUCENENET: Backport of the LUCENE-10352 guard clause. Enforcing minimumLength >= 1
+ // prevents a zero-length term (e.g. the single empty token a KeywordTokenizer emits for
+ // empty input) from reaching the term[length - 1] read in Stem() with length == 0.
+ if (minimumLength < 1)
+ throw new ArgumentOutOfRangeException(nameof(minimumLength), "minimumLength must be >=1");
this.minimumKatakanaLength = minimumLength;
this.termAttr = AddAttribute<ICharTermAttribute>();
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Compound/TestCompoundWordTokenFilter.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Compound/TestCompoundWordTokenFilter.cs
index 8ac4e49fa..a2816c233 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Compound/TestCompoundWordTokenFilter.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Compound/TestCompoundWordTokenFilter.cs
@@ -4,6 +4,7 @@
using Lucene.Net.Analysis.Core;
using Lucene.Net.Analysis.TokenAttributes;
using Lucene.Net.Analysis.Util;
+using Lucene.Net.Attributes;
using Lucene.Net.Util;
using NUnit.Framework;
using System.IO;
@@ -266,6 +267,41 @@ public virtual void TestRandomStrings()
CheckRandomData(Random, b, 1000 * RandomMultiplier);
}
+ /// <summary>
+ /// LUCENENET-specific regression test for <a href="https://github.com/apache/lucenenet/issues/1072">#1072</a>.
+ /// <para/>
+ /// When a term has leading non-letter characters (which the hyphenator counts via
+ /// <c>iIgnoreAtBeginning</c>) the resulting hyphenation points can contain two equal values,
+ /// so <see cref="HyphenationCompoundWordTokenFilter"/>'s <c>Decompose()</c> computes a
+ /// <c>partLength</c> of <c>0</c>. With <c>minSubwordSize == 0</c> that zero-length part is not
+ /// filtered out, and the genitive-'s fallback then probes the dictionary with a length of
+ /// <c>partLength - 1 == -1</c>. Upstream Java tolerates the negative length (its CharArrayMap
+ /// silently treats it as "not found"), but Lucene.NET's CharArrayMap validates its arguments
+ /// and throws <see cref="System.ArgumentOutOfRangeException"/>. The filter must not pass a
+ /// negative length downstream. The input <c>"...rindfleisch"</c> yields hyphenation points
+ /// <c>[0, 7, 11, 11]</c>, deterministically reproducing the crash that
+ /// <c>TestRandomChains</c> hit under seed <c>0xc0ab8518c470366f:0x6ed44cef961049a2</c>.
+ /// </summary>
+ [Test]
+ [LuceneNetSpecific] // Issue #1072
+ public virtual void TestZeroLengthSubwordDoesNotThrow()
+ {
+ CharArraySet dict = makeDictionary("rind", "fleisch");
+
+ using var @is = this.GetType().getResourceAsStream("da_UTF8.xml");
+ HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.GetHyphenationTree(@is);
+
+ // minSubwordSize == 0 is the configuration that lets a zero-length part through.
+ HyphenationCompoundWordTokenFilter tf = new HyphenationCompoundWordTokenFilter(TEST_VERSION_CURRENT,
+ new MockTokenizer(new StringReader("...rindfleisch"), MockTokenizer.WHITESPACE, false),
+ hyphenator, dict, CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE, 0, 100, false);
+
+ // Before the fix this threw ArgumentOutOfRangeException while decomposing. The leading
+ // "..." shifts the matched subword offsets, so no subword survives the dictionary check;
+ // the only required behavior is that decomposition completes without throwing.
+ AssertTokenStreamContents(tf, new string[] { "...rindfleisch" });
+ }
+
[Test]
public virtual void TestEmptyTerm()
{
diff --git a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Core/TestRandomChains.cs b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Core/TestRandomChains.cs
index 7abed51a2..71f8be990 100644
--- a/src/Lucene.Net.Tests.Analysis.Common/Analysis/Core/TestRandomChains.cs
+++ b/src/Lucene.Net.Tests.Analysis.Common/Analysis/Core/TestRandomChains.cs
@@ -151,6 +151,16 @@ static TestRandomChains()
foreach (ConstructorInfo ctor in c.GetConstructors())
{
brokenOffsetsConstructors[ctor] = ALWAYS;
+
+ // LUCENENET specific (#1072): Also fully exclude these broken-offsets producers
+ // from the random chains, not just relax offset validation. brokenOffsetsConstructors
+ // only flips offsetsAreCorrect=false (which suppresses ValidatingTokenFilter's checks),
+ // but the offending filter still runs and can feed backwards offsets into a downstream
+ // word-combiner (e.g. ShingleFilter), which then throws from OffsetAttribute.SetOffset.
+ // Upstream Lucene resolved this in 9.1 (LUCENE-10352) by excluding these classes via the
+ // @IgnoreRandomChains annotation. Until that annotation is backported, exclude them here.
+ // TODO: Remove this when LUCENE-10352 (IgnoreRandomChains) is backported.
+ brokenConstructors[ctor] = ALWAYS;
}
}
}
@@ -1169,8 +1179,8 @@ private TokenFilterSpec NewFilterChain(Random random, Tokenizer tokenizer, bool
// hack: MockGraph/MockLookahead has assertions that will trip if they follow
// an offsets violator. so we can't use them after e.g. wikipediatokenizer
if (!spec.offsetsAreCorrect &&
- (ctor.DeclaringType.Equals(typeof(MockGraphTokenFilter)))
- || ctor.DeclaringType.Equals(typeof(MockRandomLookaheadTokenFilter)))
+ (ctor.DeclaringType.Equals(typeof(MockGraphTokenFilter))
+ || ctor.DeclaringType.Equals(typeof(MockRandomLookaheadTokenFilter))))
{
continue;
}
@@ -1284,7 +1294,6 @@ private static TEnum RandomEnum<TEnum>(Random random)
}
[Test]
- [AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/271#issuecomment-973005744")] // LUCENENET TODO: this test occasionally fails
public void TestRandomChains_()
{
int numIterations = AtLeast(20);
@@ -1311,7 +1320,6 @@ public void TestRandomChains_()
// we might regret this decision...
[Test]
- [AwaitsFix(BugUrl = "https://github.com/apache/lucenenet/issues/271#issuecomment-973005744")] // LUCENENET TODO: this test occasionally fails
public void TestRandomChainsWithLargeStrings()
{
int numIterations = AtLeast(20);
diff --git a/src/Lucene.Net.Tests.Analysis.Kuromoji/TestJapaneseKatakanaStemFilter.cs b/src/Lucene.Net.Tests.Analysis.Kuromoji/TestJapaneseKatakanaStemFilter.cs
index fb0111cf0..0cbec3965 100644
--- a/src/Lucene.Net.Tests.Analysis.Kuromoji/TestJapaneseKatakanaStemFilter.cs
+++ b/src/Lucene.Net.Tests.Analysis.Kuromoji/TestJapaneseKatakanaStemFilter.cs
@@ -1,6 +1,7 @@
using Lucene.Net.Analysis.Core;
using Lucene.Net.Analysis.Miscellaneous;
using Lucene.Net.Analysis.Util;
+using Lucene.Net.Attributes;
using NUnit.Framework;
using System;
@@ -96,5 +97,25 @@ public void TestEmptyTerm()
CheckOneTerm(a, "", "");
}
+
+ /// <summary>
+ /// LUCENENET-specific regression test for <a href="https://github.com/apache/lucenenet/issues/1072">#1072</a>.
+ /// <para/>
+ /// Verifies the LUCENE-10352 guard: the constructor must reject a <c>minimumLength</c> less than
+ /// <c>1</c>. A <c>minimumLength</c> of <c>0</c> would let a zero-length token (such as the single empty
+ /// token a <see cref="KeywordTokenizer"/> emits for empty input) skip the length check in <c>Stem()</c>,
+ /// after which the <c>term[length - 1]</c> access reads <c>term[-1]</c> and throws. This surfaced as a
+ /// residual <c>TestRandomChains</c> failure reported on PR #1348 (e.g. seed
+ /// <c>0x951afd480395f9b7:0x63fc16274945f1a3</c>).
+ /// </summary>
+ [Test]
+ [LuceneNetSpecific] // Issue #1072
+ public void TestMinimumLengthLessThanOneThrows()
+ {
+ Tokenizer tokenizer = new KeywordTokenizer(new System.IO.StringReader(""));
+
+ Assert.Throws<ArgumentOutOfRangeException>(() => new JapaneseKatakanaStemFilter(tokenizer, minimumLength: 0));
+ Assert.Throws<ArgumentOutOfRangeException>(() => new JapaneseKatakanaStemFilter(tokenizer, minimumLength: -1));
+ }
}
}