You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2023/11/28 15:54:02 UTC

(lucene) branch branch_9x updated: Make FSTPostingsFormat load FSTs off-heap (#12552)

This is an automated email from the ASF dual-hosted git repository.

mikemccand pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new e6d6f786ac6 Make FSTPostingsFormat load FSTs off-heap (#12552)
e6d6f786ac6 is described below

commit e6d6f786ac6b63c98966b7214b4ff1b78af313ce
Author: Tony-X <do...@hotmail.com>
AuthorDate: Tue Sep 19 13:29:23 2023 -0700

    Make FSTPostingsFormat load FSTs off-heap (#12552)
    
    * Make FSTPostingsFormat load FSTs off-heap
---
 lucene/CHANGES.txt                                 |  2 ++
 .../lucene/codecs/memory/FSTTermsReader.java       | 22 +++++++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 8b27ae93a19..881f3f3000f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -181,6 +181,8 @@ Changes in runtime behavior
 
 * GITHUB#12765: Disable vectorization on VMs that are not Hotspot-based.  (Uwe Schindler, Robert Muir)
 
+* GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)
+
 Bug Fixes
 ---------------------
 
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
index 5e61ee60bb6..59d92ba5acd 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -39,6 +39,7 @@ import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -50,6 +51,7 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
 import org.apache.lucene.util.fst.BytesRefFSTEnum;
 import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
 import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.OffHeapFSTStore;
 import org.apache.lucene.util.fst.Outputs;
 import org.apache.lucene.util.fst.Util;
 
@@ -61,10 +63,10 @@ import org.apache.lucene.util.fst.Util;
  * @lucene.experimental
  */
 public class FSTTermsReader extends FieldsProducer {
-  final TreeMap<String, TermsReader> fields = new TreeMap<>();
-  final PostingsReaderBase postingsReader;
 
-  // static boolean TEST = false;
+  private final TreeMap<String, TermsReader> fields = new TreeMap<>();
+  private final PostingsReaderBase postingsReader;
+  private final IndexInput fstTermsInput;
 
   public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
       throws IOException {
@@ -73,7 +75,9 @@ public class FSTTermsReader extends FieldsProducer {
             state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
 
     this.postingsReader = postingsReader;
-    final IndexInput in = state.directory.openInput(termsFileName, state.context);
+    this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD);
+
+    IndexInput in = this.fstTermsInput;
 
     boolean success = false;
     try {
@@ -106,9 +110,7 @@ public class FSTTermsReader extends FieldsProducer {
       }
       success = true;
     } finally {
-      if (success) {
-        IOUtils.close(in);
-      } else {
+      if (success == false) {
         IOUtils.closeWhileHandlingException(in);
       }
     }
@@ -164,7 +166,7 @@ public class FSTTermsReader extends FieldsProducer {
   @Override
   public void close() throws IOException {
     try {
-      IOUtils.close(postingsReader);
+      IOUtils.close(postingsReader, fstTermsInput);
     } finally {
       fields.clear();
     }
@@ -192,7 +194,9 @@ public class FSTTermsReader extends FieldsProducer {
       this.sumTotalTermFreq = sumTotalTermFreq;
       this.sumDocFreq = sumDocFreq;
       this.docCount = docCount;
-      this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo));
+      OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
+      this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);
+      in.skipBytes(offHeapFSTStore.size());
     }
 
     @Override