You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2023/10/26 04:34:54 UTC
[lucene] branch main updated: Disable suffix sharing for block tree index (#12722)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new c701a5d9be4 Disable suffix sharing for block tree index (#12722)
c701a5d9be4 is described below
commit c701a5d9be4d8dc677ea994f8ef5fdd8945760d6
Author: gf2121 <52...@users.noreply.github.com>
AuthorDate: Thu Oct 26 12:34:48 2023 +0800
Disable suffix sharing for block tree index (#12722)
---
lucene/CHANGES.txt | 3 +++
.../codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java | 7 ++++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index cfe00abdf2e..d8347d6c03c 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -231,6 +231,9 @@ Optimizations
* GITHUB#12712: Speed up sorting postings file with an offline radix sorter in BPIndexReader. (Guo Feng)
+* GITHUB#12702: Disable suffix sharing for block tree index, making writing the terms dictionary index faster
+ and less RAM hungry, while making the index a bit (~1.X% for the terms index file on wikipedia). (Guo Feng, Mike McCandless)
+
Changes in runtime behavior
---------------------
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 0dded44d6c4..acd75092022 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -521,7 +521,12 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
final FSTCompiler<BytesRef> fstCompiler =
- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).bytesPageBits(pageBits).build();
+ new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
+ // Disable suffixes sharing for block tree index because suffixes are mostly dropped
+ // from the FST index and left in the term blocks.
+ .suffixRAMLimitMB(0d)
+ .bytesPageBits(pageBits)
+ .build();
// if (DEBUG) {
// System.out.println(" compile index for prefix=" + prefix);
// }