(lucene) branch branch_9x updated: Fix duplicate values in org.apache.lucene.analysis.ko.dict.UserDictionary (#13427)

[ch...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

chegar pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new 4be65312f7b Fix duplicate values in org.apache.lucene.analysis.ko.dict.UserDictionary (#13427)
4be65312f7b is described below

commit 4be65312f7bbd4b09cff66433ebd95b30067b690
Author: Chris Hegarty <62...@users.noreply.github.com>
AuthorDate: Mon May 27 17:53:01 2024 +0100

    Fix duplicate values in org.apache.lucene.analysis.ko.dict.UserDictionary (#13427)
    
    Remove incorrect assertion in org.apache.lucene.analysis.ko.dict.UserDictionary, and replace with array copy if duplicate values are passed.
---
 .../apache/lucene/analysis/ko/dict/UserDictionary.java  |  5 ++++-
 .../apache/lucene/analysis/ko/TestKoreanTokenizer.java  | 17 +++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
index 5614451df64..4cebdf2024c 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
 import java.util.Comparator;
 import java.util.List;
 import org.apache.lucene.analysis.ko.POS;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.IntsRefBuilder;
 import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.FSTCompiler;
@@ -138,10 +139,12 @@ public final class UserDictionary implements Dictionary {
       lastToken = token;
       ord++;
     }
+    if (entryIndex < rightIds.length) {
+      rightIds = ArrayUtil.copyOfSubArray(rightIds, 0, entryIndex);
+    }
     this.fst =
         new TokenInfoFST(FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
     this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
-    assert entryIndex == rightIds.length;
     this.rightIds = rightIds;
   }
 
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
index 315c876435c..a2349a651d0 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/TestKoreanTokenizer.java
@@ -20,6 +20,7 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
+import java.io.StringReader;
 import java.nio.charset.StandardCharsets;
 import java.util.Random;
 import org.apache.lucene.analysis.Analyzer;
@@ -593,6 +594,22 @@ public class TestKoreanTokenizer extends BaseTokenStreamTestCase {
         new POS.Tag[] {POS.Tag.SL});
   }
 
+  public void testDuplicate() throws IOException {
+    String s = "c++\nC쁠쁠\n세종\n세종\n세종시 세종 시";
+    try (Reader rulesReader = new StringReader(s)) {
+      var dict = UserDictionary.open(rulesReader);
+      assertTrue(dict.getRightId(3) != 0);
+      assertThrows(ArrayIndexOutOfBoundsException.class, () -> dict.getRightId(4));
+    }
+
+    String dupdup = "c++\nC쁠쁠\n세종\n세종\n세종시 세종 시\n세종시 세종 시";
+    try (Reader rulesReader = new StringReader(dupdup)) {
+      var dict = UserDictionary.open(rulesReader);
+      assertTrue(dict.getRightId(3) != 0);
+      assertThrows(ArrayIndexOutOfBoundsException.class, () -> dict.getRightId(4));
+    }
+  }
+
   private void assertReadings(Analyzer analyzer, String input, String... readings)
       throws IOException {
     try (TokenStream ts = analyzer.tokenStream("ignored", input)) {