You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2023/10/26 04:34:54 UTC

[lucene] branch main updated: Disable suffix sharing for block tree index (#12722)

This is an automated email from the ASF dual-hosted git repository.

guofeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new c701a5d9be4 Disable suffix sharing for block tree index (#12722)
c701a5d9be4 is described below

commit c701a5d9be4d8dc677ea994f8ef5fdd8945760d6
Author: gf2121 <52...@users.noreply.github.com>
AuthorDate: Thu Oct 26 12:34:48 2023 +0800

    Disable suffix sharing for block tree index (#12722)
---
 lucene/CHANGES.txt                                                 | 3 +++
 .../codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java    | 7 ++++++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index cfe00abdf2e..d8347d6c03c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -231,6 +231,9 @@ Optimizations
 
 * GITHUB#12712: Speed up sorting postings file with an offline radix sorter in BPIndexReader. (Guo Feng)
 
+* GITHUB#12702: Disable suffix sharing for block tree index, making writing the terms dictionary index faster
+  and less RAM hungry, while making the index a bit (~1.X% for the terms index file on wikipedia). (Guo Feng, Mike McCandless)
+
 Changes in runtime behavior
 ---------------------
 
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 0dded44d6c4..acd75092022 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -521,7 +521,12 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
 
       final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
       final FSTCompiler<BytesRef> fstCompiler =
-          new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).bytesPageBits(pageBits).build();
+          new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
+              // Disable suffixes sharing for block tree index because suffixes are mostly dropped
+              // from the FST index and left in the term blocks.
+              .suffixRAMLimitMB(0d)
+              .bytesPageBits(pageBits)
+              .build();
       // if (DEBUG) {
       //  System.out.println("  compile index for prefix=" + prefix);
       // }