You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2023/09/19 20:29:30 UTC
[lucene] branch main updated: Make FSTPostingsFormat load FSTs off-heap (#12552)
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new ca69ae6d98d Make FSTPostingsFormat load FSTs off-heap (#12552)
ca69ae6d98d is described below
commit ca69ae6d98dc1e5a250d3a6c85f016be2c6307ad
Author: Tony-X <do...@hotmail.com>
AuthorDate: Tue Sep 19 13:29:23 2023 -0700
Make FSTPostingsFormat load FSTs off-heap (#12552)
* Make FSTPostingsFormat load FSTs off-heap
---
lucene/CHANGES.txt | 2 ++
.../lucene/codecs/memory/FSTTermsReader.java | 22 +++++++++++++---------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0cab7fbf7aa..7e4a4f31cc6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -91,6 +91,8 @@ Optimizations
* GITHUB#12408: Lazy initialization improvements for Facets implementations when there are segments with no hits
to count. (Greg Miller)
+* GITHUB#12552: Make FSTPostingsFormat load FSTs off-heap. (Tony X)
+
Bug Fixes
---------------------
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
index 8a16de822e4..d4d59d278ff 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java
@@ -39,6 +39,7 @@ import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
@@ -50,6 +51,7 @@ import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.BytesRefFSTEnum.InputOutput;
import org.apache.lucene.util.fst.FST;
+import org.apache.lucene.util.fst.OffHeapFSTStore;
import org.apache.lucene.util.fst.Outputs;
import org.apache.lucene.util.fst.Util;
@@ -61,9 +63,9 @@ import org.apache.lucene.util.fst.Util;
* @lucene.experimental
*/
public class FSTTermsReader extends FieldsProducer {
- final TreeMap<String, TermsReader> fields = new TreeMap<>();
- final PostingsReaderBase postingsReader;
- // static boolean TEST = false;
+ private final TreeMap<String, TermsReader> fields = new TreeMap<>();
+ private final PostingsReaderBase postingsReader;
+ private final IndexInput fstTermsInput;
public FSTTermsReader(SegmentReadState state, PostingsReaderBase postingsReader)
throws IOException {
@@ -72,7 +74,9 @@ public class FSTTermsReader extends FieldsProducer {
state.segmentInfo.name, state.segmentSuffix, FSTTermsWriter.TERMS_EXTENSION);
this.postingsReader = postingsReader;
- final IndexInput in = state.directory.openInput(termsFileName, state.context);
+ this.fstTermsInput = state.directory.openInput(termsFileName, IOContext.LOAD);
+
+ IndexInput in = this.fstTermsInput;
boolean success = false;
try {
@@ -105,9 +109,7 @@ public class FSTTermsReader extends FieldsProducer {
}
success = true;
} finally {
- if (success) {
- IOUtils.close(in);
- } else {
+ if (success == false) {
IOUtils.closeWhileHandlingException(in);
}
}
@@ -163,7 +165,7 @@ public class FSTTermsReader extends FieldsProducer {
@Override
public void close() throws IOException {
try {
- IOUtils.close(postingsReader);
+ IOUtils.close(postingsReader, fstTermsInput);
} finally {
fields.clear();
}
@@ -191,7 +193,9 @@ public class FSTTermsReader extends FieldsProducer {
this.sumTotalTermFreq = sumTotalTermFreq;
this.sumDocFreq = sumDocFreq;
this.docCount = docCount;
- this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo));
+ OffHeapFSTStore offHeapFSTStore = new OffHeapFSTStore();
+ this.dict = new FST<>(in, in, new FSTTermOutputs(fieldInfo), offHeapFSTStore);
+ in.skipBytes(offHeapFSTStore.size());
}
@Override