You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2026/09/28 01:38:40 UTC
(lucene) branch main updated: Allow configurable buffer size in SegmentingTokenizerBase and recognize whitespace as safe boundary in ThaiTokenizer (#10153) (#16727)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new dda6bd36b81 Allow configurable buffer size in SegmentingTokenizerBase and recognize whitespace as safe boundary in ThaiTokenizer (#10153) (#16727)
dda6bd36b81 is described below
commit dda6bd36b811c82bebe78ad8cec72f2b8e7719b9
Author: Kamthorn Krairaksa <ka...@users.noreply.github.com>
AuthorDate: Mon Sep 28 08:38:34 2026 +0700
Allow configurable buffer size in SegmentingTokenizerBase and recognize whitespace as safe boundary in ThaiTokenizer (#10153) (#16727)
---
lucene/CHANGES.txt | 2 +
.../apache/lucene/analysis/th/ThaiTokenizer.java | 40 +++++++++++++++++-
.../lucene/analysis/th/ThaiTokenizerFactory.java | 4 +-
.../analysis/util/SegmentingTokenizerBase.java | 31 ++++++++++++--
.../lucene/analysis/th/TestThaiTokenizer.java | 47 ++++++++++++++++++++++
.../analysis/util/TestSegmentingTokenizerBase.java | 20 ++++++++-
6 files changed, 138 insertions(+), 6 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 2d746932314..6c4e131d58c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -230,6 +230,8 @@ Optimizations
Bug Fixes
---------------------
+* GITHUB#10153: Allow configurable buffer size in SegmentingTokenizerBase and recognize whitespace as safe boundary in ThaiTokenizer to avoid splitting words across buffer boundaries. (Kamthorn Krairaksa)
+
* GITHUB#16553: Fix stored-fields corruption when PerField#finish throws: the stored-fields frame
was left open, causing "Wrote N docs, finish called with numDocs=M" at flush. (Michael Marshall)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
index 7e76cfde780..1436981e38b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
@@ -55,6 +55,9 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
/** used for breaking the text into sentences */
private static final BreakIterator sentenceProto = BreakIterator.getSentenceInstance(Locale.ROOT);
+ /** Default buffer size in characters for Thai sentence segmentation */
+ public static final int DEFAULT_BUFFER_SIZE = BUFFERMAX;
+
private final BreakIterator wordBreaker;
private final CharArrayIterator wrapper = CharArrayIterator.newWordInstance();
@@ -82,6 +85,25 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary);
}
+ /**
+ * Creates a new ThaiTokenizer with a custom buffer size.
+ *
+ * @param bufferSize internal buffer size in characters
+ */
+ public ThaiTokenizer(int bufferSize) {
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, null, bufferSize);
+ }
+
+ /**
+ * Creates a new ThaiTokenizer with a user dictionary and custom buffer size.
+ *
+ * @param userDictionary custom dictionary of words, or null for default dictionary only
+ * @param bufferSize internal buffer size in characters
+ */
+ public ThaiTokenizer(CharArraySet userDictionary, int bufferSize) {
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary, bufferSize);
+ }
+
/** Creates a new ThaiTokenizer, supplying the AttributeFactory */
public ThaiTokenizer(AttributeFactory factory) {
this(factory, null);
@@ -94,7 +116,18 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
* @param userDictionary custom dictionary of words, or null for default dictionary only
*/
public ThaiTokenizer(AttributeFactory factory, CharArraySet userDictionary) {
- super(factory, (BreakIterator) sentenceProto.clone());
+ this(factory, userDictionary, BUFFERMAX);
+ }
+
+ /**
+ * Creates a new ThaiTokenizer, supplying the AttributeFactory, user dictionary, and buffer size.
+ *
+ * @param factory AttributeFactory to use
+ * @param userDictionary custom dictionary of words, or null for default dictionary only
+ * @param bufferSize internal buffer size in characters
+ */
+ public ThaiTokenizer(AttributeFactory factory, CharArraySet userDictionary, int bufferSize) {
+ super(factory, (BreakIterator) sentenceProto.clone(), bufferSize);
if (!DBBI_AVAILABLE) {
throw new UnsupportedOperationException(
"This JRE does not have support for Thai segmentation");
@@ -117,6 +150,11 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
}
}
+ @Override
+ protected boolean isSafeEnd(char ch) {
+ return super.isSafeEnd(ch) || Character.isWhitespace(ch);
+ }
+
@Override
protected void setNextSentence(int sentenceStart, int sentenceEnd) {
this.sentenceStart = sentenceStart;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java
index b03f10b796d..5cd1e7d3d81 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java
@@ -44,12 +44,14 @@ public class ThaiTokenizerFactory extends TokenizerFactory implements ResourceLo
public static final String NAME = "thai";
private final String dictFile;
+ private final int bufferSize;
private CharArraySet dictionary;
/** Creates a new ThaiTokenizerFactory */
public ThaiTokenizerFactory(Map<String, String> args) {
super(args);
dictFile = get(args, "dictionary");
+ bufferSize = getInt(args, "bufferSize", ThaiTokenizer.DEFAULT_BUFFER_SIZE);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -69,6 +71,6 @@ public class ThaiTokenizerFactory extends TokenizerFactory implements ResourceLo
@Override
public Tokenizer create(AttributeFactory factory) {
- return new ThaiTokenizer(factory, dictionary);
+ return new ThaiTokenizer(factory, dictionary, bufferSize);
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/SegmentingTokenizerBase.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/SegmentingTokenizerBase.java
index 9131ec3e456..227304606b0 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/SegmentingTokenizerBase.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/util/SegmentingTokenizerBase.java
@@ -36,8 +36,8 @@ import org.apache.lucene.util.AttributeFactory;
* @lucene.experimental
*/
public abstract class SegmentingTokenizerBase extends Tokenizer {
- protected static final int BUFFERMAX = 1024;
- protected final char[] buffer = new char[BUFFERMAX];
+ public static final int BUFFERMAX = 1024;
+ protected final char[] buffer;
/** true length of text in the buffer */
private int length = 0;
@@ -60,12 +60,37 @@ public abstract class SegmentingTokenizerBase extends Tokenizer {
* newly created or cloned one should always be provided to this constructor.
*/
public SegmentingTokenizerBase(BreakIterator iterator) {
- this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, iterator);
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, iterator, BUFFERMAX);
+ }
+
+ /**
+ * Construct a new SegmenterBase, using the provided BreakIterator and buffer size.
+ *
+ * @param iterator BreakIterator for sentence segmentation
+ * @param bufferSize internal buffer size in characters
+ */
+ public SegmentingTokenizerBase(BreakIterator iterator, int bufferSize) {
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, iterator, bufferSize);
}
/** Construct a new SegmenterBase, also supplying the AttributeFactory */
public SegmentingTokenizerBase(AttributeFactory factory, BreakIterator iterator) {
+ this(factory, iterator, BUFFERMAX);
+ }
+
+ /**
+ * Construct a new SegmenterBase, supplying the AttributeFactory, BreakIterator, and buffer size.
+ *
+ * @param factory AttributeFactory to use
+ * @param iterator BreakIterator for sentence segmentation
+ * @param bufferSize internal buffer size in characters
+ */
+ public SegmentingTokenizerBase(AttributeFactory factory, BreakIterator iterator, int bufferSize) {
super(factory);
+ if (bufferSize <= 0) {
+ throw new IllegalArgumentException("bufferSize must be > 0");
+ }
+ this.buffer = new char[bufferSize];
this.iterator = iterator;
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
index 264aee2598d..56780dac821 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
@@ -63,4 +63,51 @@ public class TestThaiTokenizer extends BaseTokenStreamTestCase {
tokenizer.setReader(new StringReader("ภาษาไทย"));
assertTokenStreamContents(tokenizer, new String[] {"ภาษา", "ไทย"});
}
+
+ public void testLongTextAcrossBufferBoundary() throws IOException {
+ // Generate long Thai text (> 1024 chars) with spaces as clause boundaries
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < 110; i++) {
+ sb.append("ภาษาไทย "); // 9 chars per repetition, 110 * 9 = 990 chars
+ }
+ // Now at ~990 chars; add words spanning across index 1024
+ sb.append("มหาวิทยาลัย "); // 12 chars -> total ~1002
+ sb.append("กรุงเทพมหานคร "); // 14 chars -> total ~1016
+ sb.append("ประเทศไทย"); // 9 chars -> total ~1025, crossing 1024 boundary
+
+ Tokenizer tokenizer = new ThaiTokenizer();
+ tokenizer.setReader(new StringReader(sb.toString()));
+ tokenizer.reset();
+ boolean foundCrossingWord = false;
+ org.apache.lucene.analysis.tokenattributes.CharTermAttribute termAtt =
+ tokenizer.addAttribute(org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class);
+ while (tokenizer.incrementToken()) {
+ String term = termAtt.toString();
+ if ("ประเทศ".equals(term)) {
+ foundCrossingWord = true;
+ }
+ assertFalse("Word should not be truncated into fragment 'ประ'", "ประ".equals(term));
+ assertFalse("Word should not be truncated into fragment 'เทศ'", "เทศ".equals(term));
+ }
+ tokenizer.end();
+ tokenizer.close();
+ assertTrue("Should find intact term 'ประเทศ'", foundCrossingWord);
+ }
+
+ public void testCustomBufferSize() throws IOException {
+ // Tiny buffer size of 32 chars
+ Tokenizer tokenizer = new ThaiTokenizer(32);
+ tokenizer.setReader(new StringReader("การทดสอบ ภาษาไทย มหาวิทยาลัย"));
+ assertTokenStreamContents(
+ tokenizer, new String[] {"การ", "ทดสอบ", "ภาษา", "ไทย", "มหาวิทยาลัย"});
+ }
+
+ public void testFactoryWithBufferSize() throws IOException {
+ java.util.Map<String, String> args = new java.util.HashMap<>();
+ args.put("bufferSize", "2048");
+ ThaiTokenizerFactory factory = new ThaiTokenizerFactory(args);
+ Tokenizer tokenizer = factory.create();
+ tokenizer.setReader(new StringReader("ภาษาไทย"));
+ assertTokenStreamContents(tokenizer, new String[] {"ภาษา", "ไทย"});
+ }
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
index 20af3bd0e08..3befb1024f3 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestSegmentingTokenizerBase.java
@@ -149,6 +149,20 @@ public class TestSegmentingTokenizerBase extends BaseTokenStreamTestCase {
checkRandomData(random(), sentenceAndWord, 200 * RANDOM_MULTIPLIER);
}
+ /** Test customizable buffer size with tiny buffer */
+ public void testCustomBufferSize() throws IOException {
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ return new TokenStreamComponents(new WholeSentenceTokenizer(16));
+ }
+ };
+ assertAnalyzesTo(
+ a, "Sentence one.\nSentence two.", new String[] {"Sentence one.\n", "Sentence two."});
+ a.close();
+ }
+
// some tokenizers for testing
/** silly tokenizer that just returns whole sentences as tokens */
@@ -160,7 +174,11 @@ public class TestSegmentingTokenizerBase extends BaseTokenStreamTestCase {
private OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
public WholeSentenceTokenizer() {
- super(newAttributeFactory(), BreakIterator.getSentenceInstance(Locale.ROOT));
+ this(BUFFERMAX);
+ }
+
+ public WholeSentenceTokenizer(int bufferSize) {
+ super(newAttributeFactory(), BreakIterator.getSentenceInstance(Locale.ROOT), bufferSize);
}
@Override