You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by do...@apache.org on 2023/04/23 15:44:28 UTC
[lucene] branch branch_9x updated: Hunspell: reduce suggestion set dependency on the hash table order (#12239)
This is an automated email from the ASF dual-hosted git repository.
donnerpeter pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new f4f7e13149f Hunspell: reduce suggestion set dependency on the hash table order (#12239)
f4f7e13149f is described below
commit f4f7e13149f457db658fbfd9d0a1e84b3c2ff1fc
Author: Peter Gromov <pe...@jetbrains.com>
AuthorDate: Sun Apr 23 16:51:17 2023 +0200
Hunspell: reduce suggestion set dependency on the hash table order (#12239)
When adding words to a dictionary, suggestions for other words shouldn't change unless they're directly related to the added words.
But before, GeneratingSuggester selected 100 best first matches from the hash table, whose order can change significantly after adding any unrelated word.
That resulted in unexpected suggestion changes on seemingly unrelated dictionary edits.
---
lucene/CHANGES.txt | 2 ++
.../analysis/hunspell/GeneratingSuggester.java | 7 ++++-
.../lucene/analysis/hunspell/StemmerTestBase.java | 14 +++++----
.../lucene/analysis/hunspell/TestHunspell.java | 35 ++++++++++++++++++++++
4 files changed, 51 insertions(+), 7 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 1e60b307455..a1d2ed19d44 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -111,6 +111,8 @@ Other
* GITHUB#11868: Add a FilterIndexInput and FilterIndexOutput class to more easily and safely create delegate
IndexInput and IndexOutput classes (Marc D'Mello)
+* GITHUB#12239: Hunspell: reduced suggestion set dependency on the hash table order (Peter Gromov)
+
======================== Lucene 9.5.0 =======================
API Changes
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java
index dafb1494ea5..82d58080f9b 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/GeneratingSuggester.java
@@ -98,7 +98,7 @@ class GeneratingSuggester {
sc += commonPrefix(word, rootChars) - longerWorsePenalty(word.length(), rootChars.length);
boolean overflow = roots.size() == MAX_ROOTS;
- if (overflow && sc <= roots.peek().score) {
+ if (overflow && isWorseThan(sc, rootChars, roots.peek())) {
return;
}
@@ -119,6 +119,11 @@ class GeneratingSuggester {
return roots.stream().sorted().collect(Collectors.toList());
}
+ private static boolean isWorseThan(int score, CharsRef candidate, Weighted<Root<String>> root) {
+ return score < root.score
+ || score == root.score && CharSequence.compare(candidate, root.word.word) > 0;
+ }
+
private void processSuggestibleWords(
int minLength, int maxLength, BiConsumer<CharsRef, Supplier<IntsRef>> processor) {
if (entryCache != null) {
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
index bdffa5a3dad..d8020a3ee9f 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/StemmerTestBase.java
@@ -65,13 +65,15 @@ public abstract class StemmerTestBase extends LuceneTestCase {
}
}
+ return loadDictionary(ignoreCase, affixStream, dictStreams);
+ }
+
+ protected static Dictionary loadDictionary(
+ boolean ignoreCase, InputStream affixStream, InputStream... dictStreams)
+ throws IOException, ParseException {
try {
- return new Dictionary(
- new ByteBuffersDirectory(),
- "dictionary",
- affixStream,
- Arrays.asList(dictStreams),
- ignoreCase);
+ ByteBuffersDirectory dir = new ByteBuffersDirectory();
+ return new Dictionary(dir, "dictionary", affixStream, Arrays.asList(dictStreams), ignoreCase);
} finally {
IOUtils.closeWhileHandlingException(affixStream);
IOUtils.closeWhileHandlingException(dictStreams);
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspell.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspell.java
index 4b104d487d9..f4b8cc7f917 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspell.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestHunspell.java
@@ -21,8 +21,11 @@ import static org.apache.lucene.analysis.hunspell.TimeoutPolicy.NO_TIMEOUT;
import static org.apache.lucene.analysis.hunspell.TimeoutPolicy.RETURN_PARTIAL_RESULT;
import static org.apache.lucene.analysis.hunspell.TimeoutPolicy.THROW_EXCEPTION;
+import java.io.ByteArrayInputStream;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.text.ParseException;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -239,4 +242,36 @@ public class TestHunspell extends LuceneTestCase {
private void checkCompression(Hunspell h, String expected, String... words) {
assertEquals(expected, h.compress(List.of(words)).internalsToString());
}
+
+ @Test
+ public void testSuggestionOrderStabilityOnDictionaryEditing() throws IOException, ParseException {
+ String original = "some_word";
+
+ List<String> words = new ArrayList<>();
+ for (char c = 0; c < 65535; c++) {
+ if (Character.isLetter(c)) {
+ words.add(original + c);
+ }
+ }
+
+ String smallDict = "1\n" + String.join("\n", words.subList(0, words.size() / 4));
+ String largerDict = "1\n" + String.join("\n", words);
+ Dictionary small =
+ loadDictionary(
+ false,
+ new ByteArrayInputStream(new byte[0]),
+ new ByteArrayInputStream(smallDict.getBytes(StandardCharsets.UTF_8)));
+ Dictionary larger =
+ loadDictionary(
+ false,
+ new ByteArrayInputStream(new byte[0]),
+ new ByteArrayInputStream(largerDict.getBytes(StandardCharsets.UTF_8)));
+
+ assertFalse(new Hunspell(small).spell(original));
+
+ List<String> smallSug = new Hunspell(small).suggest(original);
+ List<String> largerSug = new Hunspell(larger).suggest(original);
+ assertEquals(smallSug.toString(), 4, smallSug.size());
+ assertEquals(smallSug, largerSug);
+ }
}