You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2023/09/19 20:29:30 UTC

[lucene] branch main updated: Make FSTPostingsFormat load FSTs off-heap (#12552)

This is an automated email from the ASF dual-hosted git repository.

sokolov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new ca69ae6d98d Make FSTPostingsFormat load FSTs off-heap (#12552)
ca69ae6d98d is described below

commit ca69ae6d98dc1e5a250d3a6c85f016be2c6307ad
Author: Tony-X <do...@hotmail.com>
AuthorDate: Tue Sep 19 13:29:23 2023 -0700

    Make FSTPostingsFormat load FSTs off-heap (#12552)
    
    * Make FSTPostingsFormat load FSTs off-heap
---
 lucene/CHANGES.txt                                 |  2 ++
 .../lucene/codecs/memory/FSTTermsReader.java       | 22 +++++++++++++---------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0cab7fbf7aa..7e4a4f31cc6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -91,6 +91,8 @@ Optimizations
 * GITHUB#12408: Lazy initialization improvements for Facets implementations when there are segments with no hits
   to count. (Greg Miller)
 
+* GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)
+
 Bug Fixes
 ---------------------
 
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
index 8a16de822e4..d4d59d278ff 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -39,6 +39,7 @@ import org.apache.lucene.index.TermState;
 import org.apache.lucene.index.Terms;
 import org.apache.lucene.index.TermsEnum;
 import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
@@ -50,6 +51,7 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
 import org.apache.lucene.util.fst.BytesRefFSTEnum;
 import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
 import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.OffHeapFSTStore;
 import org.apache.lucene.util.fst.Outputs;
 import org.apache.lucene.util.fst.Util;
 
@@ -61,9 +63,9 @@ import org.apache.lucene.util.fst.Util;
  * @lucene.experimental
  */
 public class FSTTermsReader extends FieldsProducer {
-  final TreeMap<String, TermsReader> fields = new TreeMap<>();
-  final PostingsReaderBase postingsReader;
-  // static boolean TEST = false;
+  private final TreeMap<String, TermsReader> fields = new TreeMap<>();
+  private final PostingsReaderBase postingsReader;
+  private final IndexInput fstTermsInput;
 
   public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
       throws IOException {
@@ -72,7 +74,9 @@ public class FSTTermsReader extends FieldsProducer {
             state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
 
     this.postingsReader = postingsReader;
-    final IndexInput in = state.directory.openInput(termsFileName, state.context);
+    this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD);
+
+    IndexInput in = this.fstTermsInput;
 
     boolean success = false;
     try {
@@ -105,9 +109,7 @@ public class FSTTermsReader extends FieldsProducer {
       }
       success = true;
     } finally {
-      if (success) {
-        IOUtils.close(in);
-      } else {
+      if (success == false) {
         IOUtils.closeWhileHandlingException(in);
       }
     }
@@ -163,7 +165,7 @@ public class FSTTermsReader extends FieldsProducer {
   @Override
   public void close() throws IOException {
     try {
-      IOUtils.close(postingsReader);
+      IOUtils.close(postingsReader, fstTermsInput);
     } finally {
       fields.clear();
     }
@@ -191,7 +193,9 @@ public class FSTTermsReader extends FieldsProducer {
       this.sumTotalTermFreq = sumTotalTermFreq;
       this.sumDocFreq = sumDocFreq;
       this.docCount = docCount;
-      this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo));
+      OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
+      this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);
+      in.skipBytes(offHeapFSTStore.size());
     }
 
     @Override