You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2026/09/27 21:09:34 UTC
(lucene) branch main updated: Add ThaiCharFilter, ThaiNormalizationFilter, and ThaiNormalizer (#16716) (#16717)
This is an automated email from the ASF dual-hosted git repository.
rmuir pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new e64ae25dcd8 Add ThaiCharFilter, ThaiNormalizationFilter, and ThaiNormalizer (#16716) (#16717)
e64ae25dcd8 is described below
commit e64ae25dcd894019125d0a0bd723133b633b009e
Author: Kamthorn Krairaksa <ka...@users.noreply.github.com>
AuthorDate: Mon Sep 28 04:09:27 2026 +0700
Add ThaiCharFilter, ThaiNormalizationFilter, and ThaiNormalizer (#16716) (#16717)
* Add ThaiNormalizationFilter and ThaiNormalizer to Thai analysis pipeline
* Add ThaiCharFilter and ThaiCharFilterFactory for pre-tokenization normalization (#16716)
- Implements ThaiCharFilter extending BaseCharFilter to normalize character anomalies before tokenization.
- Crucial for Thai: typographic anomalies (such as double Sara E 'เเ' or separated Sara Am) cause JDK BreakIterator to fail boundary detection and incorrectly merge adjacent words. Pre-tokenization char filtering restores correct word boundaries.
- Replaces double Sara E ('เ'+'เ' -> 'แ'), recomposes Sara Am, removes Zero-Width characters, deduplicates marks, and correctly records offset corrections via addOffCorrectMap.
- Adds ThaiCharFilterFactory (SPI: 'thai') registered in CharFilterFactory service.
- Wraps Reader with ThaiCharFilter in ThaiAnalyzer (initReader and initReaderForNormalization).
- Adds TestThaiCharFilter and TestThaiCharFilterFactory.
* Update CHANGES.txt for #16717
* Apply spotless formatting to Thai CharFilter and Normalizer
* Fix: register ThaiCharFilterFactory and ThaiNormalizationFilterFactory in module-info.java for JPMS discovery
---
lucene/CHANGES.txt | 4 +
lucene/analysis/common/src/java/module-info.java | 4 +-
.../apache/lucene/analysis/th/ThaiAnalyzer.java | 18 +-
.../apache/lucene/analysis/th/ThaiCharFilter.java | 211 +++++++++++++++++++++
.../lucene/analysis/th/ThaiCharFilterFactory.java | 64 +++++++
.../analysis/th/ThaiNormalizationFilter.java | 50 +++++
.../th/ThaiNormalizationFilterFactory.java | 64 +++++++
.../apache/lucene/analysis/th/ThaiNormalizer.java | 175 +++++++++++++++++
.../org.apache.lucene.analysis.CharFilterFactory | 1 +
.../org.apache.lucene.analysis.TokenFilterFactory | 1 +
.../lucene/analysis/th/TestThaiAnalyzer.java | 19 ++
.../lucene/analysis/th/TestThaiCharFilter.java | 133 +++++++++++++
.../analysis/th/TestThaiCharFilterFactory.java | 43 +++++
.../th/TestThaiNormalizationFilterFactory.java | 49 +++++
.../lucene/analysis/th/TestThaiNormalizer.java | 110 +++++++++++
15 files changed, 942 insertions(+), 4 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 1df64189326..7e267cfbb04 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -123,6 +123,10 @@ API Changes
New Features
---------------------
+* GITHUB#16717: Add ThaiCharFilter, ThaiCharFilterFactory, and ThaiNormalizationFilter for
+ pre-tokenization character normalization and token-level normalization in ThaiAnalyzer.
+ (Kamthorn Krairaksa)
+
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
modern Hangul conjoining jamo before KoreanTokenizer. (Mingi Jeong)
diff --git a/lucene/analysis/common/src/java/module-info.java b/lucene/analysis/common/src/java/module-info.java
index 025ea08c6df..ac8be52b084 100644
--- a/lucene/analysis/common/src/java/module-info.java
+++ b/lucene/analysis/common/src/java/module-info.java
@@ -150,7 +150,8 @@ module org.apache.lucene.analysis.common {
org.apache.lucene.analysis.charfilter.MappingCharFilterFactory,
org.apache.lucene.analysis.cjk.CJKWidthCharFilterFactory,
org.apache.lucene.analysis.fa.PersianCharFilterFactory,
- org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory;
+ org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory,
+ org.apache.lucene.analysis.th.ThaiCharFilterFactory;
provides org.apache.lucene.analysis.TokenFilterFactory with
org.apache.lucene.analysis.tr.ApostropheFilterFactory,
org.apache.lucene.analysis.ar.ArabicNormalizationFilterFactory,
@@ -263,6 +264,7 @@ module org.apache.lucene.analysis.common {
org.apache.lucene.analysis.core.FlattenGraphFilterFactory,
org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory,
org.apache.lucene.analysis.te.TeluguStemFilterFactory,
+ org.apache.lucene.analysis.th.ThaiNormalizationFilterFactory,
org.apache.lucene.analysis.tr.TurkishLowerCaseFilterFactory,
org.apache.lucene.analysis.util.ElisionFilterFactory;
provides org.apache.lucene.analysis.TokenizerFactory with
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
index 12595982f9a..715237810af 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
@@ -94,15 +94,15 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {
* the text in the provided {@link Reader}.
*
* @return {@link org.apache.lucene.analysis.Analyzer.TokenStreamComponents} built from a {@link
- * ThaiTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter} and {@link
- * StopFilter}
+ * ThaiTokenizer} filtered with {@link LowerCaseFilter}, {@link DecimalDigitFilter}, {@link
+ * ThaiNormalizationFilter} and {@link StopFilter}
*/
@Override
protected TokenStreamComponents createComponents(String fieldName) {
final Tokenizer source = new ThaiTokenizer();
TokenStream result = new LowerCaseFilter(source);
result = new DecimalDigitFilter(result);
-
+ result = new ThaiNormalizationFilter(result);
result = new StopFilter(result, stopwords);
return new TokenStreamComponents(source, result);
}
@@ -111,6 +111,18 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {
protected TokenStream normalize(String fieldName, TokenStream in) {
TokenStream result = new LowerCaseFilter(in);
result = new DecimalDigitFilter(result);
+ result = new ThaiNormalizationFilter(result);
return result;
}
+
+ /** Wraps the Reader with {@link ThaiCharFilter} */
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ThaiCharFilter(reader);
+ }
+
+ @Override
+ protected Reader initReaderForNormalization(String fieldName, Reader reader) {
+ return new ThaiCharFilter(reader);
+ }
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilter.java
new file mode 100644
index 00000000000..6577a6b1d3e
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilter.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.IOException;
+import java.io.Reader;
+import org.apache.lucene.analysis.charfilter.BaseCharFilter;
+
+/**
+ * A {@link org.apache.lucene.analysis.CharFilter} that normalizes Thai text at the character stream
+ * level before tokenization.
+ *
+ * <p>Applying normalization before tokenization is critical for Thai, because typographic anomalies
+ * (such as two consecutive Sara E characters instead of Sara Ae, or zero-width characters) prevent
+ * the dictionary-based {@link java.text.BreakIterator} from locating word boundaries, causing
+ * multiple adjacent words to be mistakenly merged into one token.
+ *
+ * <p>Normalizations performed:
+ *
+ * <ul>
+ * <li>Removes Zero-Width characters (U+200B ZWSP, U+200C ZWNJ, U+200D ZWJ, U+FEFF BOM)
+ * <li>Replaces double Sara E (เ + เ, U+0E40 U+0E40) with Sara Ae (แ, U+0E41)
+ * <li>Recomposes decomposed Sara Am (U+0E4D + U+0E32 → U+0E33, and U+0E4D + tone + U+0E32
+ * → tone + U+0E33)
+ * <li>Deduplicates consecutive repeated diacritics / tone marks (e.g. ดีี → ดี)
+ * <li>Reorders misplaced tone mark before an above/below vowel to canonical order
+ * <li>Replaces Lakkhangyao (ๅ, U+0E45) with Sara Aa (า, U+0E32) unless preceded by Ru (ฤ) or Lu
+ * (ฦ)
+ * </ul>
+ *
+ * <p>All character deletions and contractions correctly update the character offset map using
+ * {@link #addOffCorrectMap(int, int)}.
+ *
+ * @since 11.0.0
+ */
+public class ThaiCharFilter extends BaseCharFilter {
+
+ private final char[] inputBuffer = new char[8];
+ private int inBufLen = 0;
+ private boolean eof = false;
+
+ private int outputOff = 0;
+ private int cumulativeDiff = 0;
+ private char prevChar = 0;
+ private char pendingOutputChar = 0;
+
+ /**
+ * Creates a new ThaiCharFilter wrapping the given {@link Reader}.
+ *
+ * @param in the input reader
+ */
+ public ThaiCharFilter(Reader in) {
+ super(in);
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (pendingOutputChar != 0) {
+ char c = pendingOutputChar;
+ pendingOutputChar = 0;
+ prevChar = c;
+ outputOff++;
+ return c;
+ }
+
+ while (true) {
+ fillBuffer();
+
+ if (inBufLen == 0) {
+ return -1;
+ }
+
+ char c0 = inputBuffer[0];
+
+ // 1. Zero-width character removal (U+200B, U+200C, U+200D, U+FEFF)
+ if (c0 == '\u200B' || c0 == '\u200C' || c0 == '\u200D' || c0 == '\uFEFF') {
+ shiftBuffer(1);
+ cumulativeDiff++;
+ addOffCorrectMap(outputOff, cumulativeDiff);
+ continue;
+ }
+
+ // 2. Deduplicate consecutive identical diacritics / tone marks
+ if (isThaiDiacritic(c0) && c0 == prevChar) {
+ shiftBuffer(1);
+ cumulativeDiff++;
+ addOffCorrectMap(outputOff, cumulativeDiff);
+ continue;
+ }
+
+ // 3. Double Sara E (เ + เ -> แ)
+ if (c0 == '\u0E40' && inBufLen >= 2 && inputBuffer[1] == '\u0E40') {
+ shiftBuffer(2);
+ cumulativeDiff++;
+ addOffCorrectMap(outputOff + 1, cumulativeDiff);
+ prevChar = '\u0E41';
+ outputOff++;
+ return '\u0E41';
+ }
+
+ // 4. Sara Am recomposition
+ if (c0 == '\u0E4D') {
+ // Subcase 4A: Nikhahit + Sara Aa (ํ + า -> ำ)
+ if (inBufLen >= 2 && inputBuffer[1] == '\u0E32') {
+ shiftBuffer(2);
+ cumulativeDiff++;
+ addOffCorrectMap(outputOff + 1, cumulativeDiff);
+ prevChar = '\u0E33';
+ outputOff++;
+ return '\u0E33';
+ }
+ // Subcase 4B: Nikhahit + Tone + Sara Aa (ํ + tone + า -> tone + ำ)
+ if (inBufLen >= 3 && isThaiTone(inputBuffer[1]) && inputBuffer[2] == '\u0E32') {
+ char tone = inputBuffer[1];
+ shiftBuffer(3);
+ pendingOutputChar = '\u0E33';
+ cumulativeDiff++;
+ addOffCorrectMap(outputOff + 2, cumulativeDiff);
+ prevChar = tone;
+ outputOff++;
+ return tone;
+ }
+ }
+
+ // 5. Misplaced tone mark before above/below vowel: swap to canonical order
+ if (isThaiTone(c0) && inBufLen >= 2 && isThaiAboveBelowVowel(inputBuffer[1])) {
+ char vowel = inputBuffer[1];
+ char tone = c0;
+ inputBuffer[0] = vowel;
+ inputBuffer[1] = tone;
+ // Continue loop to process the swapped vowel at index 0
+ continue;
+ }
+
+ // 6. Lakkhangyao (ๅ, U+0E45) -> Sara Aa (า, U+0E32) unless preceded by Ru (ฤ) or Lu (ฦ)
+ if (c0 == '\u0E45') {
+ if (prevChar != '\u0E24' && prevChar != '\u0E26') {
+ c0 = '\u0E32';
+ }
+ }
+
+ // Default: emit character c0
+ shiftBuffer(1);
+ prevChar = c0;
+ outputOff++;
+ return c0;
+ }
+ }
+
+ @Override
+ public int read(char[] cbuf, int off, int len) throws IOException {
+ if (len <= 0) {
+ return 0;
+ }
+ int numRead = 0;
+ for (int i = off; i < off + len; i++) {
+ int c = read();
+ if (c == -1) {
+ break;
+ }
+ cbuf[i] = (char) c;
+ numRead++;
+ }
+ return numRead == 0 ? -1 : numRead;
+ }
+
+ private void fillBuffer() throws IOException {
+ while (!eof && inBufLen < 4) {
+ int ch = input.read();
+ if (ch == -1) {
+ eof = true;
+ break;
+ }
+ inputBuffer[inBufLen++] = (char) ch;
+ }
+ }
+
+ private void shiftBuffer(int count) {
+ int remaining = inBufLen - count;
+ if (remaining > 0) {
+ System.arraycopy(inputBuffer, count, inputBuffer, 0, remaining);
+ }
+ inBufLen = remaining;
+ }
+
+ private static boolean isThaiDiacritic(char c) {
+ return c == '\u0E31' || (c >= '\u0E34' && c <= '\u0E3A') || (c >= '\u0E47' && c <= '\u0E4E');
+ }
+
+ private static boolean isThaiTone(char c) {
+ return (c >= '\u0E48' && c <= '\u0E4B') || c == '\u0E4C';
+ }
+
+ private static boolean isThaiAboveBelowVowel(char c) {
+ return c == '\u0E31' || (c >= '\u0E34' && c <= '\u0E3A') || c == '\u0E47';
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilterFactory.java
new file mode 100644
index 00000000000..dac127d4311
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiCharFilterFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.Reader;
+import java.util.Map;
+import org.apache.lucene.analysis.CharFilterFactory;
+
+/**
+ * Factory for {@link ThaiCharFilter}.
+ *
+ * <pre class="prettyprint">
+ * <fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <charFilter name="thai"/>
+ * <tokenizer name="thai"/>
+ * </analyzer>
+ * </fieldType></pre>
+ *
+ * @since 11.0.0
+ * @lucene.spi {@value #NAME}
+ */
+public class ThaiCharFilterFactory extends CharFilterFactory {
+
+ /** SPI name */
+ public static final String NAME = "thai";
+
+ /** Default constructor for SPI */
+ public ThaiCharFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ /** Default constructor */
+ public ThaiCharFilterFactory() {
+ this(Map.of());
+ }
+
+ @Override
+ public Reader create(Reader input) {
+ return new ThaiCharFilter(input);
+ }
+
+ @Override
+ public Reader normalize(Reader input) {
+ return create(input);
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilter.java
new file mode 100644
index 00000000000..53f6c3dd471
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+
+/**
+ * A {@link TokenFilter} that applies {@link ThaiNormalizer} to normalize Thai characters.
+ *
+ * @see ThaiNormalizer
+ */
+public final class ThaiNormalizationFilter extends TokenFilter {
+
+ private final ThaiNormalizer normalizer = new ThaiNormalizer();
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
+
+ public ThaiNormalizationFilter(TokenStream input) {
+ super(input);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (input.incrementToken()) {
+ if (!keywordAtt.isKeyword()) {
+ termAtt.setLength(normalizer.normalize(termAtt.buffer(), termAtt.length()));
+ }
+ return true;
+ }
+ return false;
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilterFactory.java
new file mode 100644
index 00000000000..a5ab4a0cd29
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizationFilterFactory.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.util.Map;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Factory for {@link ThaiNormalizationFilter}.
+ *
+ * <pre><code class="language-xml">
+ * <fieldType name="text_thnormal" class="solr.TextField" positionIncrementGap="100">
+ * <analyzer>
+ * <tokenizer class="solr.ThaiTokenizerFactory"/>
+ * <filter class="solr.ThaiNormalizationFilterFactory"/>
+ * </analyzer>
+ * </fieldType></code></pre>
+ *
+ * @since 11.0.0
+ * @lucene.spi {@value #NAME}
+ */
+public class ThaiNormalizationFilterFactory extends TokenFilterFactory {
+
+ /** SPI name */
+ public static final String NAME = "thaiNormalization";
+
+ /** Creates a new ThaiNormalizationFilterFactory */
+ public ThaiNormalizationFilterFactory(Map<String, String> args) {
+ super(args);
+ if (!args.isEmpty()) {
+ throw new IllegalArgumentException("Unknown parameters: " + args);
+ }
+ }
+
+ /** Default ctor for compatibility with SPI */
+ public ThaiNormalizationFilterFactory() {
+ throw defaultCtorException();
+ }
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new ThaiNormalizationFilter(input);
+ }
+
+ @Override
+ public TokenStream normalize(TokenStream input) {
+ return create(input);
+ }
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizer.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizer.java
new file mode 100644
index 00000000000..5ce1c08fa7b
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiNormalizer.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import static org.apache.lucene.analysis.util.StemmerUtil.delete;
+
+/**
+ * Normalizer for Thai.
+ *
+ * <p>Normalizes Thai text to handle common orthographic variations, typos, and encoding anomalies:
+ *
+ * <ul>
+ * <li>Removes zero-width characters (ZWSP U+200B, ZWNJ U+200C).
+ * <li>Replaces double Sara E (U+0E40 U+0E40) with Sara Ae (U+0E41).
+ * <li>Recomposes decomposed Sara Am (Nikhahit U+0E4D + optional tone mark + Sara Aa U+0E32) into
+ * canonical Sara Am (U+0E33).
+ * <li>Normalizes Lakkhangyao (U+0E45) to Sara Aa (U+0E32) unless preceded by Ru (U+0E24) or Lu
+ * (U+0E26).
+ * <li>Reorders misplaced tone marks (U+0E48-U+0E4B) or Thanthakhat (U+0E4C) with above/below
+ * vowels (U+0E31, U+0E34-U+0E3A, U+0E47) to canonical Unicode order (vowel before tone mark).
+ * <li>Reorders follow vowels (e.g. Sara Aa U+0E32, Sara Am U+0E33) and misplaced trailing tone
+ * marks.
+ * <li>Deduplicates repeated vowels, tone marks, and diacritics.
+ * <li>Removes dangling combining marks at the start of a token.
+ * </ul>
+ */
+public class ThaiNormalizer {
+
+ /**
+ * Normalize an input buffer of Thai text.
+ *
+ * @param s input buffer
+ * @param len length of input buffer
+ * @return length of input buffer after normalization
+ */
+ public int normalize(char[] s, int len) {
+ if (len == 0) {
+ return 0;
+ }
+
+ // Step 1: Remove zero-width characters
+ for (int i = 0; i < len; i++) {
+ if (s[i] == '\u200B' || s[i] == '\u200C') {
+ len = delete(s, i, len);
+ i--;
+ }
+ }
+
+ // Remove leading dangling non-base marks at token start
+ while (len > 0 && isDanglingMark(s[0])) {
+ len = delete(s, 0, len);
+ }
+
+ // Step 2: Handle compositions (Double Sara E -> Sara Ae, Sara Am recomposition, Lakkhangyao)
+ for (int i = 0; i < len; i++) {
+ char c = s[i];
+
+ // Double Sara E -> Sara Ae
+ if (c == '\u0E40' && i + 1 < len && s[i + 1] == '\u0E40') {
+ s[i] = '\u0E41';
+ len = delete(s, i + 1, len);
+ continue;
+ }
+
+ // Nikhahit (U+0E4D) + optional tone mark + Sara Aa (U+0E32) -> optional tone mark + Sara Am
+ // (U+0E33)
+ if (c == '\u0E4D') {
+ if (i + 1 < len && s[i + 1] == '\u0E32') {
+ s[i] = '\u0E33';
+ len = delete(s, i + 1, len);
+ continue;
+ } else if (i + 2 < len && isToneMark(s[i + 1]) && s[i + 2] == '\u0E32') {
+ // Nikhahit + Tone + Sara Aa -> Tone + Sara Am
+ char tone = s[i + 1];
+ s[i] = tone;
+ s[i + 1] = '\u0E33';
+ len = delete(s, i + 2, len);
+ continue;
+ }
+ }
+
+ // Tone mark + Nikhahit + Sara Aa -> Tone + Sara Am
+ if (isToneMark(c) && i + 2 < len && s[i + 1] == '\u0E4D' && s[i + 2] == '\u0E32') {
+ s[i + 1] = '\u0E33';
+ len = delete(s, i + 2, len);
+ continue;
+ }
+
+ // Lakkhangyao (U+0E45) -> Sara Aa (U+0E32) unless preceded by Ru (U+0E24) or Lu (U+0E26)
+ if (c == '\u0E45') {
+ if (i == 0 || (s[i - 1] != '\u0E24' && s[i - 1] != '\u0E26')) {
+ s[i] = '\u0E32';
+ }
+ }
+ }
+
+ // Step 3: Canonical reordering
+ // Standard order: Consonant -> Above/Below Vowel -> Tone mark / Thanthakhat
+ // Also: Follow vowel (Sara Aa / Sara Am) + Tone mark -> Tone mark + Follow vowel
+ boolean changed = true;
+ while (changed) {
+ changed = false;
+ for (int i = 0; i < len - 1; i++) {
+ // Tone mark / Thanthakhat followed by Above/Below Vowel
+ if (isToneMarkOrThanthakhat(s[i]) && isAboveOrBelowVowel(s[i + 1])) {
+ char tmp = s[i];
+ s[i] = s[i + 1];
+ s[i + 1] = tmp;
+ changed = true;
+ }
+ // Follow vowel followed by Tone mark (e.g., Sara Am + Tone -> Tone + Sara Am)
+ else if (isFollowVowel(s[i]) && isToneMark(s[i + 1])) {
+ char tmp = s[i];
+ s[i] = s[i + 1];
+ s[i + 1] = tmp;
+ changed = true;
+ }
+ }
+ }
+
+ // Step 4: Deduplicate repeated vowels, tone marks, and diacritics
+ for (int i = 0; i < len - 1; i++) {
+ // Identical consecutive combining mark or vowel
+ if (isDeduplicable(s[i]) && s[i] == s[i + 1]) {
+ len = delete(s, i + 1, len);
+ i--;
+ }
+ // If two different consecutive tone marks appear, keep the last one
+ else if (isToneMark(s[i]) && isToneMark(s[i + 1])) {
+ len = delete(s, i, len);
+ i--;
+ }
+ }
+
+ return len;
+ }
+
+ private static boolean isToneMark(char c) {
+ return c >= '\u0E48' && c <= '\u0E4B';
+ }
+
+ private static boolean isToneMarkOrThanthakhat(char c) {
+ return isToneMark(c) || c == '\u0E4C';
+ }
+
+ private static boolean isAboveOrBelowVowel(char c) {
+ return c == '\u0E31' || (c >= '\u0E34' && c <= '\u0E3A') || c == '\u0E47';
+ }
+
+ private static boolean isFollowVowel(char c) {
+ return c == '\u0E30' || c == '\u0E32' || c == '\u0E33' || c == '\u0E45';
+ }
+
+ private static boolean isDanglingMark(char c) {
+ return isAboveOrBelowVowel(c) || isToneMarkOrThanthakhat(c) || c == '\u0E4D' || c == '\u0E4E';
+ }
+
+ private static boolean isDeduplicable(char c) {
+ return (c >= '\u0E30' && c <= '\u0E3A') || c == '\u0E45' || (c >= '\u0E47' && c <= '\u0E4E');
+ }
+}
diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory
index c9f43ec2df7..2851b7f152a 100644
--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory
+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.CharFilterFactory
@@ -18,3 +18,4 @@ org.apache.lucene.analysis.charfilter.MappingCharFilterFactory
org.apache.lucene.analysis.cjk.CJKWidthCharFilterFactory
org.apache.lucene.analysis.fa.PersianCharFilterFactory
org.apache.lucene.analysis.pattern.PatternReplaceCharFilterFactory
+org.apache.lucene.analysis.th.ThaiCharFilterFactory
diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
index 1b33f2603b3..b5b276803bf 100644
--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
@@ -124,5 +124,6 @@ org.apache.lucene.analysis.synonym.word2vec.Word2VecSynonymFilterFactory
org.apache.lucene.analysis.core.FlattenGraphFilterFactory
org.apache.lucene.analysis.te.TeluguNormalizationFilterFactory
org.apache.lucene.analysis.te.TeluguStemFilterFactory
+org.apache.lucene.analysis.th.ThaiNormalizationFilterFactory
org.apache.lucene.analysis.tr.TurkishLowerCaseFilterFactory
org.apache.lucene.analysis.util.ElisionFilterFactory
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
index bd8c9f8440a..9bca83c8f7b 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
@@ -146,4 +146,23 @@ public class TestThaiAnalyzer extends BaseTokenStreamTestCase {
new int[] {4, 7, 9, 14, 19, 22, 25, 29, 33, 36, 39, 41});
analyzer.close();
}
+
+ public void testNormalization() throws Exception {
+ Analyzer analyzer = new ThaiAnalyzer(CharArraySet.EMPTY_SET);
+ // Double Sara E -> Sara Ae
+ assertAnalyzesTo(
+ analyzer, "\u0E40\u0E40\u0E1B\u0E25\u0E01", new String[] {"\u0E41\u0E1B\u0E25\u0E01"});
+ // Decomposed Sara Am -> Sara Am (pre-tokenization allows BreakIterator to properly segment ทำ
+ // and งาน)
+ assertAnalyzesTo(
+ analyzer,
+ "\u0E17\u0E4D\u0E32\u0E07\u0E32\u0E19",
+ new String[] {"\u0E17\u0E33", "\u0E07\u0E32\u0E19"});
+ // Sentence with double Sara E properly segmented into individual words thanks to ThaiCharFilter
+ assertAnalyzesTo(
+ analyzer, "ฉันรัก\u0E40\u0E40มวมาก", new String[] {"ฉัน", "รัก", "แมว", "มาก"});
+ // Duplicate vowels
+ assertAnalyzesTo(analyzer, "\u0E14\u0E35\u0E35", new String[] {"\u0E14\u0E35"});
+ analyzer.close();
+ }
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilter.java
new file mode 100644
index 00000000000..c8901bf6b34
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilter.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.tests.analysis.MockTokenizer;
+
+/** Test case for {@link ThaiCharFilter}. */
+public class TestThaiCharFilter extends BaseTokenStreamTestCase {
+
+ /** Double Sara E (เ + เ) normalized to Sara Ae (แ) with offset correction */
+ public void testDoubleSaraE() throws IOException {
+ CharFilter reader = new ThaiCharFilter(new StringReader("เเมว"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"แมว"}, new int[] {0}, new int[] {4}, 4);
+ }
+
+ /** Sara Am recomposition (ํ + า -> ำ, and ํ + tone + า -> tone + ำ) */
+ public void testSaraAmRecomposition() throws IOException {
+ // น + ํ + ้ + า (4 chars) -> น้ำ (3 chars)
+ CharFilter reader = new ThaiCharFilter(new StringReader("น\u0E4D\u0E49\u0E32"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"น้ำ"}, new int[] {0}, new int[] {4}, 4);
+
+ // ท + ํ + า (3 chars) -> ทำ (2 chars)
+ reader = new ThaiCharFilter(new StringReader("ท\u0E4D\u0E32"));
+ ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"ทำ"}, new int[] {0}, new int[] {3}, 3);
+ }
+
+ /** Consecutive repeated diacritics / tone marks deduplication */
+ public void testDuplicateDiacritics() throws IOException {
+ // ด + ี + ี (3 chars) -> ดี (2 chars)
+ CharFilter reader = new ThaiCharFilter(new StringReader("ดีี"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"ดี"}, new int[] {0}, new int[] {3}, 3);
+
+ // ไ + ม + ้ + ้ (4 chars) -> ไม้ (3 chars)
+ reader = new ThaiCharFilter(new StringReader("ไม้้"));
+ ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"ไม้"}, new int[] {0}, new int[] {4}, 4);
+ }
+
+ /** Zero-width space and joiner removal */
+ public void testZeroWidth() throws IOException {
+ // สวัส + \u200B + ดี (7 chars in input) -> สวัสดี (6 chars in output, endOffset 7)
+ CharFilter reader = new ThaiCharFilter(new StringReader("สวัส\u200Bดี"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"สวัสดี"}, new int[] {0}, new int[] {7}, 7);
+ }
+
+ /** Misplaced tone mark before above/below vowel swapped to canonical order */
+ public void testToneReordering() throws IOException {
+ // บ + ่ + ั + น -> บ + ั + ่ + น (บั่น)
+ CharFilter reader = new ThaiCharFilter(new StringReader("บ\u0E48\u0E31น"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"บั่น"}, new int[] {0}, new int[] {4}, 4);
+ }
+
+ /** Lakkhangyao (ๅ) converted to Sara Aa (า) except after Ru (ฤ) or Lu (ฦ) */
+ public void testLakkhangyao() throws IOException {
+ CharFilter reader = new ThaiCharFilter(new StringReader("ฤๅ ฦๅ กๅ"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(
+ ts, new String[] {"ฤๅ", "ฦๅ", "กา"}, new int[] {0, 3, 6}, new int[] {2, 5, 8}, 8);
+ }
+
+ /**
+ * Demonstrates that pre-tokenization char filtering allows BreakIterator to segment words
+ * correctly that would otherwise be mistakenly merged into one huge token.
+ */
+ public void testBreakIteratorSegmentationWithCharFilter() throws IOException {
+ assumeTrue(
+ "JRE does not support Thai dictionary-based BreakIterator", ThaiTokenizer.DBBI_AVAILABLE);
+ // "ฉันรักเเมวมาก" (contains double Sara E in เเมว)
+ // Without ThaiCharFilter, BreakIterator lumps "รักเเมวมาก" into 1 token.
+ // With ThaiCharFilter, it normalizes to "ฉันรักแมวมาก" and segments all 4 words properly.
+ Reader reader = new ThaiCharFilter(new StringReader("ฉันรักเเมวมาก"));
+ Tokenizer tokenizer = new ThaiTokenizer();
+ tokenizer.setReader(reader);
+ assertTokenStreamContents(
+ tokenizer,
+ new String[] {"ฉัน", "รัก", "แมว", "มาก"},
+ new int[] {0, 3, 6, 10},
+ new int[] {3, 6, 10, 13},
+ 13);
+ }
+
+ public void testEmpty() throws IOException {
+ CharFilter reader = new ThaiCharFilter(new StringReader(""));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {});
+ }
+
+ public void testRandom() throws IOException {
+ Analyzer analyzer =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+
+ @Override
+ protected Reader initReader(String fieldName, Reader reader) {
+ return new ThaiCharFilter(reader);
+ }
+ };
+ checkRandomData(random(), analyzer, 200 * RANDOM_MULTIPLIER);
+ analyzer.close();
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilterFactory.java
new file mode 100644
index 00000000000..83974e93775
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiCharFilterFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.Reader;
+import java.io.StringReader;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;
+
+/** Simple tests to ensure the Thai char filter factory is working. */
+public class TestThaiCharFilterFactory extends BaseTokenStreamFactoryTestCase {
+
+ public void testCharFilter() throws Exception {
+ Reader reader = charFilterFactory("Thai").create(new StringReader("เเมว"));
+ TokenStream ts = whitespaceMockTokenizer(reader);
+ assertTokenStreamContents(ts, new String[] {"แมว"});
+ }
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ charFilterFactory("Thai", "bogusArg", "bogusValue");
+ });
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizationFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizationFilterFactory.java
new file mode 100644
index 00000000000..594f3dd0500
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizationFilterFactory.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.Reader;
+import java.io.StringReader;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.tests.analysis.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.tests.analysis.MockTokenizer;
+
+/** Simple tests to ensure the Thai normalization factory is working. */
+public class TestThaiNormalizationFilterFactory extends BaseTokenStreamFactoryTestCase {
+
+ public void testNormalization() throws Exception {
+ Reader reader =
+ new StringReader("\u0E40\u0E40\u0E1B\u0E25\u0E01 \u0E17\u0E4D\u0E32\u0E07\u0E32\u0E19");
+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ ((Tokenizer) stream).setReader(reader);
+ stream = tokenFilterFactory("ThaiNormalization").create(stream);
+ assertTokenStreamContents(
+ stream, new String[] {"\u0E41\u0E1B\u0E25\u0E01", "\u0E17\u0E33\u0E07\u0E32\u0E19"});
+ }
+
+ /** Test that bogus arguments result in exception */
+ public void testBogusArguments() throws Exception {
+ IllegalArgumentException expected =
+ expectThrows(
+ IllegalArgumentException.class,
+ () -> {
+ tokenFilterFactory("ThaiNormalization", "bogusArg", "bogusValue");
+ });
+ assertTrue(expected.getMessage().contains("Unknown parameters"));
+ }
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizer.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizer.java
new file mode 100644
index 00000000000..8b760582446
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiNormalizer.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.th;
+
+import java.io.IOException;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
+
+/** Test case for {@link ThaiNormalizer} and {@link ThaiNormalizationFilter}. */
+public class TestThaiNormalizer extends BaseTokenStreamTestCase {
+
+ public void testDoubleSaraE() throws IOException {
+ // Two Sara E (U+0E40 U+0E40) should become Sara Ae (U+0E41)
+ check("\u0E40\u0E40\u0E1B\u0E25\u0E01", "\u0E41\u0E1B\u0E25\u0E01"); // เเปลก -> แปลก
+ }
+
+ public void testSaraAmDecomposition() throws IOException {
+ // Nikhahit (U+0E4D) + Sara Aa (U+0E32) -> Sara Am (U+0E33)
+ check(
+ "\u0E17\u0E4D\u0E32\u0E07\u0E32\u0E19",
+ "\u0E17\u0E33\u0E07\u0E32\u0E19"); // ทํางาน -> ทำงาน
+
+ // Nikhahit + Tone (U+0E49) + Sara Aa -> Tone + Sara Am
+ check("\u0E19\u0E4D\u0E49\u0E32", "\u0E19\u0E49\u0E33"); // นํ้า -> น้ำ
+
+ // Tone + Nikhahit + Sara Aa -> Tone + Sara Am
+ check("\u0E19\u0E49\u0E4D\u0E32", "\u0E19\u0E49\u0E33"); // น้ำ
+ }
+
+ public void testSaraAmWithFollowedTone() throws IOException {
+ // Sara Am followed by Tone mark -> Tone mark + Sara Am
+ check("\u0E19\u0E33\u0E49", "\u0E19\u0E49\u0E33");
+ }
+
+ public void testReorderToneAndAboveVowel() throws IOException {
+ // Tone mark before Above Vowel (Mai Tho before Sara I) -> Sara I before Mai Tho
+ check("\u0E01\u0E49\u0E34", "\u0E01\u0E34\u0E49"); // ก้ิ -> กิ้
+ }
+
+ public void testDuplicateVowelsAndTones() throws IOException {
+ // Duplicate Sara Ii
+ check("\u0E14\u0E35\u0E35", "\u0E14\u0E35"); // ดีี -> ดี
+ // Duplicate Mai Tho
+ check("\u0E41\u0E21\u0E49\u0E49", "\u0E41\u0E21\u0E49"); // แม้้ -> แม้
+ // Duplicate Sara Aa
+ check("\u0E19\u0E32\u0E32", "\u0E19\u0E32"); // นาา -> นา
+ }
+
+ public void testConsecutiveDifferentTones() throws IOException {
+ // When multiple different tone marks are typed, keep the last one
+ check("\u0E44\u0E21\u0E48\u0E49", "\u0E44\u0E21\u0E49"); // ไม่้ -> ไม้
+ }
+
+ public void testZeroWidthCharacters() throws IOException {
+ check(
+ "\u0E20\u0E32\u0E29\u0E32\u200B\u0E44\u0E17\u0E22",
+ "\u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22");
+ check(
+ "\u0E20\u0E32\u0E29\u0E32\u200C\u0E44\u0E17\u0E22",
+ "\u0E20\u0E32\u0E29\u0E32\u0E44\u0E17\u0E22");
+ }
+
+ public void testLakkhangyao() throws IOException {
+ // Preceded by Ru: preserve
+ check("\u0E24\u0E45", "\u0E24\u0E45");
+ // Not preceded by Ru or Lu: normalize to Sara Aa
+ check("\u0E01\u0E45", "\u0E01\u0E32");
+ }
+
+ public void testDanglingMark() throws IOException {
+ // Leading dangling tone mark without consonant
+ check("\u0E49\u0E01\u0E32", "\u0E01\u0E32");
+ }
+
+ public void testEmptyTerm() throws IOException {
+ Analyzer a =
+ new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new KeywordTokenizer();
+ return new TokenStreamComponents(tokenizer, new ThaiNormalizationFilter(tokenizer));
+ }
+ };
+ checkOneTerm(a, "", "");
+ a.close();
+ }
+
+ private void check(String input, String output) throws IOException {
+ Tokenizer tokenizer = whitespaceMockTokenizer(input);
+ TokenFilter tf = new ThaiNormalizationFilter(tokenizer);
+ assertTokenStreamContents(tf, new String[] {output});
+ }
+}