You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by do...@apache.org on 2023/11/21 07:05:48 UTC

(lucene) branch main updated: hunspell: a couple micro-optimizations to speed up dictionary loading (#12825)

This is an automated email from the ASF dual-hosted git repository.

donnerpeter pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new c23b3b33015 hunspell: a couple micro-optimizations to speed up dictionary loading (#12825)
c23b3b33015 is described below

commit c23b3b3301573e0915e771b9b4c0e593685b37bb
Author: Peter Gromov <pe...@jetbrains.com>
AuthorDate: Tue Nov 21 08:05:42 2023 +0100

    hunspell: a couple micro-optimizations to speed up dictionary loading (#12825)
    
    * hunspell: a couple micro-optimizations to speed up dictionary loading
    
    1. sort by the whole entry without searching for separators first: WordStorage doesn't require strong lexicographic order (only something close to it), and the separators are anyway before any usual word characters
    2. avoid stream overhead when adding an entry
---
 lucene/CHANGES.txt                                 |  2 ++
 .../lucene/analysis/hunspell/Dictionary.java       | 38 ++--------------------
 .../lucene/analysis/hunspell/WordStorage.java      |  9 +++--
 3 files changed, 10 insertions(+), 39 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c1980f5611c..9661d13e706 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -88,6 +88,8 @@ Optimizations
 
 * GITHUB#11857, GITHUB#11859, GITHUB#11893, GITHUB#11909: Hunspell: improved suggestion performance (Peter Gromov)
 
+* GITHUB#12825: Hunspell: improved dictionary loading performance (Peter Gromov)
+
 * GITHUB#12372: Reduce allocation during HNSW construction (Jonathan Ellis)
 
 * GITHUB#12408: Lazy initialization improvements for Facets implementations when there are segments with no hits
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index c197b16a7b7..d453404bb54 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -56,6 +56,7 @@ import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefComparator;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.IntsRefBuilder;
@@ -1087,42 +1088,7 @@ public class Dictionary {
 
   private String sortWordsOffline(
       Directory tempDir, String tempFileNamePrefix, IndexOutput unsorted) throws IOException {
-    OfflineSorter sorter =
-        new OfflineSorter(
-            tempDir,
-            tempFileNamePrefix,
-            new Comparator<>() {
-              final BytesRef scratch1 = new BytesRef();
-              final BytesRef scratch2 = new BytesRef();
-
-              private void initScratch(BytesRef o, BytesRef scratch) {
-                scratch.bytes = o.bytes;
-                scratch.offset = o.offset;
-                scratch.length = o.length;
-
-                for (int i = scratch.length - 1; i >= 0; i--) {
-                  if (scratch.bytes[scratch.offset + i] == FLAG_SEPARATOR
-                      || scratch.bytes[scratch.offset + i] == MORPH_SEPARATOR) {
-                    scratch.length = i;
-                    break;
-                  }
-                }
-              }
-
-              @Override
-              public int compare(BytesRef o1, BytesRef o2) {
-                initScratch(o1, scratch1);
-                initScratch(o2, scratch2);
-
-                int cmp = scratch1.compareTo(scratch2);
-                if (cmp == 0) {
-                  // tie break on whole row
-                  return o1.compareTo(o2);
-                } else {
-                  return cmp;
-                }
-              }
-            });
+    var sorter = new OfflineSorter(tempDir, tempFileNamePrefix, BytesRefComparator.NATURAL);
 
     String sorted;
     boolean success = false;
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java
index 0b5e4b86978..1e739f03d47 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/WordStorage.java
@@ -350,16 +350,19 @@ abstract class WordStorage {
 
       currentOrds.clear();
       boolean hasNonHidden = false;
+      boolean isSuggestible = false;
       for (char[] flags : group) {
         if (!hasFlag(flags, Dictionary.HIDDEN_FLAG)) {
           hasNonHidden = true;
-          break;
+        }
+        if (!hasNoSuggestFlag(flags)) {
+          isSuggestible = true;
         }
       }
 
       for (int i = 0; i < group.size(); i++) {
         char[] flags = group.get(i);
-        if (hasNonHidden && hasFlag(flags, Dictionary.HIDDEN_FLAG)) {
+        if (hasNonHidden && group.size() > 1 && hasFlag(flags, Dictionary.HIDDEN_FLAG)) {
           continue;
         }
 
@@ -388,7 +391,7 @@ abstract class WordStorage {
 
       int mask =
           (prevCode == 0 ? 0 : COLLISION_MASK)
-              | (group.stream().anyMatch(flags -> !hasNoSuggestFlag(flags)) ? SUGGESTIBLE_MASK : 0)
+              | (isSuggestible ? SUGGESTIBLE_MASK : 0)
               | Math.min(currentEntry.length(), MAX_STORED_LENGTH);
       hashTable[hash] = (mask << OFFSET_BITS) | pos;