(lucene) branch branch_10x updated: hunspell: improve tolerateAffixRuleCountMismatches() for common problems (#14239)

[rm...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rmuir pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_10x by this push:
     new 24057d851d9 hunspell: improve tolerateAffixRuleCountMismatches() for common problems (#14239)
24057d851d9 is described below

commit 24057d851d9c02867886750d1740aad4104e7195
Author: Robert Muir <rm...@apache.org>
AuthorDate: Sat Feb 15 08:42:03 2025 -0500

    hunspell: improve tolerateAffixRuleCountMismatches() for common problems (#14239)
    
    When the user opts-in to tolerateAffixRuleCountMismatches(), it tolerates
    some, but not all affix mismatches. Support two cases:
    
    "EOF case": count is wrong and then we encounter end of file.
    previously the exception looked like this:
    
      java.text.ParseException: Premature end of rules for SFX B Y 2
    
    "stampede case": in this case count is wrong and another instruction
    follows (such as another affix's rule).
    Previously, the exception looked like this:
    
      java.text.ParseException: Affix rule mismatch. Header: SFX A Y 2; rule: SFX B Y 2
---
 lucene/CHANGES.txt                                            |  3 +++
 .../java/org/apache/lucene/analysis/hunspell/Dictionary.java  | 11 +++++++++++
 .../org/apache/lucene/analysis/hunspell/TestDictionary.java   | 11 +++++++++++
 .../org/apache/lucene/analysis/hunspell/common-errors.aff     |  9 +++++++++
 .../org/apache/lucene/analysis/hunspell/common-errors.dic     |  9 +++++++++
 5 files changed, 43 insertions(+)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 982c3b67dce..7712a87b536 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -48,6 +48,9 @@ Improvements
   the wrapped analyzer's strategy to decide if components can be reused or need
   to be updated. (Mayya Sharipova)
 
+* GITHUB#14239: Hunspell's option to tolerate affix rule count mismatches was
+  improved to tolerate more instances of this problem.  (Robert Muir)
+
 
 Optimizations
 ---------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index fc5353560b5..4d1b7a8192e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -708,8 +708,15 @@ public class Dictionary {
     affixData = ArrayUtil.grow(affixData, currentAffix * 4 + numLines * 4);
 
     for (int i = 0; i < numLines; i++) {
+      if (tolerateAffixRuleCountMismatches()) {
+        // push back line if counts are wrong, with a limit: forgiveness is finite.
+        reader.mark(256);
+      }
       String line = reader.readLine();
       if (line == null) {
+        if (tolerateAffixRuleCountMismatches()) {
+          return; // can happen due to rule count mismatches, EOF
+        }
         throw new ParseException("Premature end of rules for " + header, reader.getLineNumber());
       }
 
@@ -717,6 +724,10 @@ public class Dictionary {
       String[] ruleArgs = splitBySpace(reader, line, 4, Integer.MAX_VALUE);
 
       if (!ruleArgs[1].equals(args[1])) {
+        if (tolerateAffixRuleCountMismatches()) {
+          reader.reset(); // can happen due to count mismatch: push back what we found
+          return;
+        }
         throw new ParseException(
             "Affix rule mismatch. Header: " + header + "; rule: " + line, reader.getLineNumber());
       }
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
index 1aab0164121..481a3dc0aae 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestDictionary.java
@@ -188,6 +188,17 @@ public class TestDictionary extends LuceneTestCase {
     loadForgivingDictionary("forgivable-errors-num.aff", "single-word.dic");
   }
 
+  /** simple tests for dictionary problems seen in the wild */
+  public void testCommonForgivableErrors() throws Exception {
+    Dictionary dictionary = loadForgivingDictionary("common-errors.aff", "common-errors.dic");
+    // try to ensure we still parsed the affixes correctly, despite the problems
+    String expectedSuffixes[] = {"ing", "ed"};
+    for (String suffix : expectedSuffixes) {
+      char reversed[] = new StringBuilder(suffix).reverse().toString().toCharArray();
+      assertNotNull("checking for " + suffix, dictionary.lookupSuffix(reversed));
+    }
+  }
+
   private Dictionary loadDictionary(String aff, String dic) throws IOException, ParseException {
     try (InputStream affixStream = getClass().getResourceAsStream(aff);
         InputStream dicStream = getClass().getResourceAsStream(dic);
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/common-errors.aff b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/common-errors.aff
new file mode 100644
index 00000000000..6b061e99c16
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/common-errors.aff
@@ -0,0 +1,9 @@
+SET UTF-8
+
+# add some suffix off-by-ones here, feel free to add more
+# SFX A: wrong count, come up short on entries and stampede into SFX B.
+# SFX B: wrong count, hit EOF
+SFX A Y 2
+SFX A 0 ing .
+SFX B Y 2
+SFX B 0 ed .
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/common-errors.dic b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/common-errors.dic
new file mode 100644
index 00000000000..48506b98960
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/common-errors.dic
@@ -0,0 +1,9 @@
+# Dictionary for testing common mistakes to tolerate in dictionaries.
+# Purpose is to test that we still behave well even with the common mistakes.
+# Lucene parser seems to be really strict with these numbers.
+# I wonder if we should just ignore them entirely and parse in a different way?
+# Have the suspicion that maybe C hunspell just uses them as a "hint" to size data?
+# Add more flags if you need to the "testword" for any more affix bugs found.
+# Or add more words if the problem is instead around this file.
+1
+testword/AB