You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2023/11/28 15:54:02 UTC
(lucene) branch branch_9x updated: Make FSTPostingsFormat load FSTs off-heap (#12552)
This is an automated email from the ASF dual-hosted git repository.
mikemccand pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new e6d6f786ac6 Make FSTPostingsFormat load FSTs off-heap (#12552)
e6d6f786ac6 is described below
commit e6d6f786ac6b63c98966b7214b4ff1b78af313ce
Author: Tony-X <do...@hotmail.com>
AuthorDate: Tue Sep 19 13:29:23 2023 -0700
Make FSTPostingsFormat load FSTs off-heap (#12552)
* Make FSTPostingsFormat load FSTs off-heap
---
lucene/CHANGES.txt | 2 ++
.../lucene/codecs/memory/FSTTermsReader.java | 22 +++++++++++++---------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 8b27ae93a19..881f3f3000f 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -181,6 +181,8 @@ Changes in runtime behavior
* GITHUB#12765: Disable vectorization on VMs that are not Hotspot-based. (Uwe Schindler, Robert Muir)
+* GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)
+
Bug Fixes
---------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
index 5e61ee60bb6..59d92ba5acd 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -39,6 +39,7 @@ import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -50,6 +51,7 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.OffHeapFSTStore;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.Util;
@@ -61,10 +63,10 @@ import org.apache.lucene.util.fst.Util;
* @lucene.experimental
*/
public class FSTTermsReader extends FieldsProducer {
- final TreeMap<String, TermsReader> fields = new TreeMap<>();
- final PostingsReaderBase postingsReader;
- // static boolean TEST = false;
+ private final TreeMap<String, TermsReader> fields = new TreeMap<>();
+ private final PostingsReaderBase postingsReader;
+ private final IndexInput fstTermsInput;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
throws IOException {
@@ -73,7 +75,9 @@ public class FSTTermsReader extends FieldsProducer {
state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
this.postingsReader = postingsReader;
- final IndexInput in = state.directory.openInput(termsFileName, state.context);
+ this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD);
+
+ IndexInput in = this.fstTermsInput;
boolean success = false;
try {
@@ -106,9 +110,7 @@ public class FSTTermsReader extends FieldsProducer {
}
success = true;
} finally {
- if (success) {
- IOUtils.close(in);
- } else {
+ if (success == false) {
IOUtils.closeWhileHandlingException(in);
}
}
@@ -164,7 +166,7 @@ public class FSTTermsReader extends FieldsProducer {
@Override
public void close() throws IOException {
try {
- IOUtils.close(postingsReader);
+ IOUtils.close(postingsReader, fstTermsInput);
} finally {
fields.clear();
}
@@ -192,7 +194,9 @@ public class FSTTermsReader extends FieldsProducer {
this.sumTotalTermFreq = sumTotalTermFreq;
this.sumDocFreq = sumDocFreq;
this.docCount = docCount;
- this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo));
+ OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
+ this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);
+ in.skipBytes(offHeapFSTStore.size());
}
@Override