You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2026/09/28 00:59:04 UTC
(lucene) branch main updated: Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory (#16721) (#16722)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 6b44b4c2a3a Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory (#16721) (#16722)
6b44b4c2a3a is described below
commit 6b44b4c2a3afadee94c035d680d8dc9856ee93d7
Author: Kamthorn Krairaksa <ka...@users.noreply.github.com>
AuthorDate: Mon Sep 28 07:58:58 2026 +0700
Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory (#16721) (#16722)
* Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory (#16721)
* Update CHANGES.txt for #16722
* Apply spotless formatting to ThaiTokenizer and TestThaiTokenizerFactory
* Simplify user dictionary min/max word length calculation per review suggestion
---
lucene/CHANGES.txt | 3 +
.../apache/lucene/analysis/th/ThaiTokenizer.java | 76 +++++++++++++++++++++-
.../lucene/analysis/th/ThaiTokenizerFactory.java | 21 +++++-
.../lucene/analysis/th/TestThaiTokenizer.java | 66 +++++++++++++++++++
.../analysis/th/TestThaiTokenizerFactory.java | 10 +++
.../lucene/analysis/th/customThaiDictionary.txt | 3 +
6 files changed, 174 insertions(+), 5 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c77499b2f48..2d746932314 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -123,6 +123,9 @@ API Changes
New Features
---------------------
+* GITHUB#16722: Support user dictionary in ThaiTokenizer and ThaiTokenizerFactory.
+ (Kamthorn Krairaksa)
+
* GITHUB#16717: Add ThaiCharFilter, ThaiCharFilterFactory, and ThaiNormalizationFilter for
pre-tokenization character normalization and token-level normalization in ThaiAnalyzer.
(Kamthorn Krairaksa)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
index 50674f1f657..7e76cfde780 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizer.java
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.th;
import java.text.BreakIterator;
import java.util.Locale;
+import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.util.CharArrayIterator;
@@ -25,7 +26,11 @@ import org.apache.lucene.analysis.util.SegmentingTokenizerBase;
import org.apache.lucene.util.AttributeFactory;
/**
- * Tokenizer that use {@link BreakIterator} to tokenize Thai text.
+ * Tokenizer that uses {@link BreakIterator} to tokenize Thai text.
+ *
+ * <p>Supports an optional user dictionary ({@link CharArraySet}) for custom or domain-specific
+ * words. When a user dictionary word matches, it takes precedence over default segmentation
+ * boundaries.
*
* <p>WARNING: this tokenizer may not be supported by all JREs. It is known to work with Sun/Oracle
* and Harmony JREs. If your application needs to be fully portable, consider using ICUTokenizer
@@ -53,6 +58,10 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
private final BreakIterator wordBreaker;
private final CharArrayIterator wrapper = CharArrayIterator.newWordInstance();
+ private final CharArraySet userDictionary;
+ private final int minDictWordLen;
+ private final int maxDictWordLen;
+
int sentenceStart;
int sentenceEnd;
@@ -61,17 +70,51 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
/** Creates a new ThaiTokenizer */
public ThaiTokenizer() {
- this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY);
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, null);
+ }
+
+ /**
+ * Creates a new ThaiTokenizer with a user dictionary.
+ *
+ * @param userDictionary custom dictionary of words, or null for default dictionary only
+ */
+ public ThaiTokenizer(CharArraySet userDictionary) {
+ this(DEFAULT_TOKEN_ATTRIBUTE_FACTORY, userDictionary);
}
/** Creates a new ThaiTokenizer, supplying the AttributeFactory */
public ThaiTokenizer(AttributeFactory factory) {
+ this(factory, null);
+ }
+
+ /**
+ * Creates a new ThaiTokenizer, supplying the AttributeFactory and user dictionary.
+ *
+ * @param factory AttributeFactory to use
+ * @param userDictionary custom dictionary of words, or null for default dictionary only
+ */
+ public ThaiTokenizer(AttributeFactory factory, CharArraySet userDictionary) {
super(factory, (BreakIterator) sentenceProto.clone());
if (!DBBI_AVAILABLE) {
throw new UnsupportedOperationException(
"This JRE does not have support for Thai segmentation");
}
wordBreaker = (BreakIterator) proto.clone();
+ this.userDictionary = userDictionary;
+ if (userDictionary != null && !userDictionary.isEmpty()) {
+ int minLen = Integer.MAX_VALUE;
+ int maxLen = 0;
+ for (Object obj : userDictionary) {
+ int length = ((char[]) obj).length;
+ minLen = Math.min(minLen, length);
+ maxLen = Math.max(maxLen, length);
+ }
+ this.minDictWordLen = minLen;
+ this.maxDictWordLen = maxLen;
+ } else {
+ this.minDictWordLen = 0;
+ this.maxDictWordLen = 0;
+ }
}
@Override
@@ -102,10 +145,39 @@ public class ThaiTokenizer extends SegmentingTokenizerBase {
return false; // BreakIterator exhausted
}
+ boolean reanchor = false;
+ if (userDictionary != null && !userDictionary.isEmpty()) {
+ int remaining = sentenceEnd - (sentenceStart + start);
+ int maxLen = Math.min(maxDictWordLen, remaining);
+ for (int l = maxLen; l >= minDictWordLen; l--) {
+ if (userDictionary.contains(buffer, sentenceStart + start, l)) {
+ int customEnd = start + l;
+ if (customEnd != end) {
+ end = customEnd;
+ if (end >= sentenceEnd - sentenceStart) {
+ wordBreaker.last();
+ } else if (wordBreaker.isBoundary(end)) {
+ wordBreaker.following(end - 1);
+ } else {
+ reanchor = true;
+ }
+ }
+ break;
+ }
+ }
+ }
+
clearAttributes();
termAtt.copyBuffer(buffer, sentenceStart + start, end - start);
offsetAtt.setOffset(
correctOffset(offset + sentenceStart + start), correctOffset(offset + sentenceStart + end));
+
+ if (reanchor) {
+ sentenceStart += end;
+ wrapper.setText(buffer, sentenceStart, sentenceEnd - sentenceStart);
+ wordBreaker.setText(wrapper);
+ }
+
return true;
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java
index ef0969681a3..b03f10b796d 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiTokenizerFactory.java
@@ -16,10 +16,14 @@
*/
package org.apache.lucene.analysis.th;
+import java.io.IOException;
import java.util.Map;
+import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.TokenizerFactory;
import org.apache.lucene.util.AttributeFactory;
+import org.apache.lucene.util.ResourceLoader;
+import org.apache.lucene.util.ResourceLoaderAware;
/**
* Factory for {@link ThaiTokenizer}.
@@ -27,21 +31,25 @@ import org.apache.lucene.util.AttributeFactory;
* <pre><code class="language-xml">
* <fieldType name="text_thai" class="solr.TextField" positionIncrementGap="100">
* <analyzer>
- * <tokenizer class="solr.ThaiTokenizerFactory"/>
+ * <tokenizer class="solr.ThaiTokenizerFactory" dictionary="custom_words.txt"/>
* </analyzer>
* </fieldType></code></pre>
*
* @since 4.10.0
* @lucene.spi {@value #NAME}
*/
-public class ThaiTokenizerFactory extends TokenizerFactory {
+public class ThaiTokenizerFactory extends TokenizerFactory implements ResourceLoaderAware {
/** SPI name */
public static final String NAME = "thai";
+ private final String dictFile;
+ private CharArraySet dictionary;
+
/** Creates a new ThaiTokenizerFactory */
public ThaiTokenizerFactory(Map<String, String> args) {
super(args);
+ dictFile = get(args, "dictionary");
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -52,8 +60,15 @@ public class ThaiTokenizerFactory extends TokenizerFactory {
throw defaultCtorException();
}
+ @Override
+ public void inform(ResourceLoader loader) throws IOException {
+ if (dictFile != null) {
+ dictionary = getWordSet(loader, dictFile, false);
+ }
+ }
+
@Override
public Tokenizer create(AttributeFactory factory) {
- return new ThaiTokenizer(factory);
+ return new ThaiTokenizer(factory, dictionary);
}
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
new file mode 100644
index 00000000000..264aee2598d
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizer.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.List;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
+
+/** Test case for {@link ThaiTokenizer}. */
+public class TestThaiTokenizer extends BaseTokenStreamTestCase {
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ assumeTrue(
+ "JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);
+ }
+
+ public void testDefaultSegmentation() throws IOException {
+ Tokenizer tokenizer = new ThaiTokenizer();
+ tokenizer.setReader(new StringReader("ภาษาไทย"));
+ assertTokenStreamContents(tokenizer, new String[] {"ภาษา", "ไทย"});
+ }
+
+ public void testUserDictionary() throws IOException {
+ CharArraySet userDict = new CharArraySet(List.of("พารากอน", "คนขับรถ"), false);
+ Tokenizer tokenizer = new ThaiTokenizer(userDict);
+ tokenizer.setReader(new StringReader("ไปพารากอนกัน"));
+ assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน", "กัน"});
+ }
+
+ public void testMultipleUserDictionaryTermsAndOffsets() throws IOException {
+ CharArraySet userDict = new CharArraySet(List.of("พารากอน", "คนขับรถ"), false);
+ Tokenizer tokenizer = new ThaiTokenizer(userDict);
+ tokenizer.setReader(new StringReader("เขาเป็นคนขับรถไปพารากอน"));
+ assertTokenStreamContents(
+ tokenizer,
+ new String[] {"เขา", "เป็น", "คนขับรถ", "ไป", "พารากอน"},
+ new int[] {0, 3, 7, 14, 16},
+ new int[] {3, 7, 14, 16, 23});
+ }
+
+ public void testEmptyUserDictionary() throws IOException {
+ CharArraySet userDict = new CharArraySet(0, false);
+ Tokenizer tokenizer = new ThaiTokenizer(userDict);
+ tokenizer.setReader(new StringReader("ภาษาไทย"));
+ assertTokenStreamContents(tokenizer, new String[] {"ภาษา", "ไทย"});
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java
index e77a749564f..21c13c7d36b 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiTokenizerFactory.java
@@ -32,6 +32,16 @@ public class TestThaiTokenizerFactory extends BaseTokenStreamFactoryTestCase {
tokenizer, new String[] {"การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"});
}
+ public void testCustomDictionary() throws Exception {
+ assumeTrue(
+ "JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);
+ Tokenizer tokenizer =
+ tokenizerFactory("Thai", "dictionary", "customThaiDictionary.txt")
+ .create(newAttributeFactory());
+ tokenizer.setReader(new StringReader("ไปพารากอน"));
+ assertTokenStreamContents(tokenizer, new String[] {"ไป", "พารากอน"});
+ }
+
/** Test that bogus arguments result in exception */
public void testBogusArguments() throws Exception {
assumeTrue(
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/customThaiDictionary.txt b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/customThaiDictionary.txt
new file mode 100644
index 00000000000..d042b89e41e
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/customThaiDictionary.txt
@@ -0,0 +1,3 @@
+# Custom Thai Dictionary
+พารากอน
+คนขับรถ