You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2024/02/07 18:16:45 UTC
(lucene) branch branch_9x updated: Make FSTCompiler.compile() to only return the FSTMetadata (#12831)
This is an automated email from the ASF dual-hosted git repository.
mikemccand pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new f3e2929a52c Make FSTCompiler.compile() to only return the FSTMetadata (#12831)
f3e2929a52c is described below
commit f3e2929a52c78413b1bdd8ec703edee7a48dee4a
Author: Dzung Bui <du...@gmail.com>
AuthorDate: Tue Feb 6 00:58:56 2024 +0900
Make FSTCompiler.compile() to only return the FSTMetadata (#12831)
* Make FSTCompiler.compile() to only return the FSTMetadata
* tidy code
---
lucene/CHANGES.txt | 4 ++
.../analysis/charfilter/NormalizeCharMap.java | 2 +-
.../apache/lucene/analysis/hunspell/ConvTable.java | 2 +-
.../lucene/analysis/hunspell/Dictionary.java | 2 +-
.../miscellaneous/StemmerOverrideFilter.java | 3 +-
.../apache/lucene/analysis/synonym/SynonymMap.java | 2 +-
.../lucene/analysis/ja/dict/UserDictionary.java | 4 +-
.../ja/util/TokenInfoDictionaryBuilder.java | 2 +-
.../lucene/analysis/ko/dict/UserDictionary.java | 3 +-
.../ko/util/TokenInfoDictionaryBuilder.java | 2 +-
.../blocktree/Lucene40BlockTreeTermsWriter.java | 2 +-
.../BooleanPerceptronClassifier.java | 2 +-
.../blockterms/VariableGapTermsIndexWriter.java | 2 +-
.../blocktreeords/OrdsBlockTreeTermsWriter.java | 2 +-
.../lucene/codecs/memory/FSTTermsWriter.java | 3 +-
.../codecs/simpletext/SimpleTextFieldsReader.java | 2 +-
.../lucene/codecs/uniformsplit/FSTDictionary.java | 3 +-
.../blocktree/Lucene90BlockTreeTermsWriter.java | 2 +-
.../src/java/org/apache/lucene/util/fst/FST.java | 19 +++++--
.../org/apache/lucene/util/fst/FSTCompiler.java | 62 +++++++++++++++-------
.../org/apache/lucene/util/fst/package-info.java | 2 +-
.../test/org/apache/lucene/util/fst/Test2BFST.java | 6 +--
.../apache/lucene/util/fst/Test2BFSTOffHeap.java | 12 ++---
.../lucene/util/fst/TestFSTDirectAddressing.java | 4 +-
.../test/org/apache/lucene/util/fst/TestFSTs.java | 40 +++++++-------
.../test/org/apache/lucene/util/fst/TestUtil.java | 2 +-
.../org/apache/lucene/demo/knn/KnnVectorDict.java | 2 +-
.../apache/lucene/misc/util/fst/TestFSTsMisc.java | 4 +-
.../idversion/VersionBlockTreeTermsWriter.java | 2 +-
.../suggest/analyzing/AnalyzingSuggester.java | 2 +-
.../suggest/analyzing/FreeTextSuggester.java | 2 +-
.../suggest/document/NRTSuggesterBuilder.java | 3 +-
.../search/suggest/fst/FSTCompletionBuilder.java | 2 +-
.../search/suggest/fst/WFSTCompletionLookup.java | 2 +-
.../apache/lucene/tests/util/fst/FSTTester.java | 29 +++++-----
35 files changed, 149 insertions(+), 90 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index f5010d29680..854f0dbe03d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -16,6 +16,10 @@ API Changes
* GITHUB#12854: Mark DrillSideways#createDrillDownFacetsCollector as @Deprecated. (Greg Miller)
+* GITHUB#12624, GITHUB#12831: Allow FSTCompiler to stream to any DataOutput while building, and
+ make compile() only return the FSTMetadata. For on-heap (default) use case, please use
+ FST.fromFSTReader(fstMetadata, fstCompiler.getFSTReader()) to create the FST. (Anh Dung Bui)
+
New Features
---------------------
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
index 01baea12b01..1ffa071835e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/NormalizeCharMap.java
@@ -111,7 +111,7 @@ public class NormalizeCharMap {
for (Map.Entry<String, String> ent : pendingPairs.entrySet()) {
fstCompiler.add(Util.toUTF16(ent.getKey(), scratch), new CharsRef(ent.getValue()));
}
- map = fstCompiler.compile();
+ map = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
pendingPairs.clear();
} catch (IOException ioe) {
// Bogus FST IOExceptions!! (will never happen)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java
index f22bee1db0d..acbf1976d58 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/ConvTable.java
@@ -51,7 +51,7 @@ class ConvTable {
fstCompiler.add(scratchInts.get(), new CharsRef(entry.getValue()));
}
- fst = fstCompiler.compile();
+ fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
} catch (IOException bogus) {
throw new RuntimeException(bogus);
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
index f8ced9055df..9d23aca9a6e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
@@ -657,7 +657,7 @@ public class Dictionary {
}
fstCompiler.add(scratch.get(), output);
}
- return fstCompiler.compile();
+ return FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
/**
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
index b46f8f8ff02..83724af63d5 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/StemmerOverrideFilter.java
@@ -222,7 +222,8 @@ public final class StemmerOverrideFilter extends TokenFilter {
intsSpare.copyUTF8Bytes(bytesRef);
fstCompiler.add(intsSpare.get(), new BytesRef(outputValues.get(id)));
}
- return new StemmerOverrideMap(fstCompiler.compile(), ignoreCase);
+ return new StemmerOverrideMap(
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()), ignoreCase);
}
}
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
index 22ba92ff555..7d08424c27a 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymMap.java
@@ -291,7 +291,7 @@ public class SynonymMap {
fstCompiler.add(Util.toUTF32(input, scratchIntsRef), scratch.toBytesRef());
}
- FST<BytesRef> fst = fstCompiler.compile();
+ FST<BytesRef> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
return new SynonymMap(fst, words, maxHorizontalContext);
}
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
index 521fa997881..506b4de980f 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/UserDictionary.java
@@ -150,7 +150,9 @@ public final class UserDictionary implements Dictionary {
segmentations.add(wordIdAndLength);
ord++;
}
- this.fst = new TokenInfoFST(fstCompiler.compile(), false);
+ this.fst =
+ new TokenInfoFST(
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()), false);
this.data = data.toArray(new String[data.size()]);
this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
}
diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
index 74645a2fc65..2189bba7060 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryBuilder.java
@@ -131,7 +131,7 @@ class TokenInfoDictionaryBuilder {
dictionary.addMapping((int) ord, offset);
offset = next;
}
- dictionary.setFST(fstCompiler.compile());
+ dictionary.setFST(FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
return dictionary;
}
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
index 7d850624530..64a06cd1abe 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/UserDictionary.java
@@ -137,7 +137,8 @@ public final class UserDictionary implements Dictionary {
lastToken = token;
ord++;
}
- this.fst = new TokenInfoFST(fstCompiler.compile());
+ this.fst =
+ new TokenInfoFST(FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
this.segmentations = segmentations.toArray(new int[segmentations.size()][]);
this.rightIds = new short[rightIds.size()];
for (int i = 0; i < rightIds.size(); i++) {
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
index 921b47ae401..6838ff08cdb 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/util/TokenInfoDictionaryBuilder.java
@@ -126,7 +126,7 @@ class TokenInfoDictionaryBuilder {
dictionary.addMapping((int) ord, offset);
offset = next;
}
- dictionary.setFST(fstCompiler.compile());
+ dictionary.setFST(FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
return dictionary;
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java
index 4f889d0914c..2ededb9391f 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene40/blocktree/Lucene40BlockTreeTermsWriter.java
@@ -498,7 +498,7 @@ public final class Lucene40BlockTreeTermsWriter extends FieldsConsumer {
}
}
- index = fstCompiler.compile();
+ index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
assert subIndices == null;
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
index 280e82cc2c0..e4c4ccb1a95 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/BooleanPerceptronClassifier.java
@@ -216,7 +216,7 @@ public class BooleanPerceptronClassifier implements Classifier<Boolean> {
fstCompiler.add(
Util.toIntsRef(scratchBytes.get(), scratchInts), entry.getValue().longValue());
}
- fst = fstCompiler.compile();
+ fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
@Override
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
index 04e3e80c71b..70f1f9918dd 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/VariableGapTermsIndexWriter.java
@@ -283,7 +283,7 @@ public class VariableGapTermsIndexWriter extends TermsIndexWriterBase {
@Override
public void finish(long termsFilePointer) throws IOException {
- fst = fstCompiler.compile();
+ fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
if (fst != null) {
fst.save(out, out);
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
index 986c6e47d4c..a7ef5ef9932 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
@@ -425,7 +425,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
assert sumTotalTermCount == totFloorTermCount;
- index = fstCompiler.compile();
+ index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
assert subIndices == null;
/*
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
index fa46f6451da..1e8c442ccc6 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsWriter.java
@@ -277,7 +277,8 @@ public class FSTTermsWriter extends FieldsConsumer {
public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) throws IOException {
// save FST dict
if (numTerms > 0) {
- final FST<FSTTermOutputs.TermData> fst = fstCompiler.compile();
+ final FST<FSTTermOutputs.TermData> fst =
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
fields.add(
new FieldMetaData(fieldInfo, numTerms, sumTotalTermFreq, sumDocFreq, docCount, fst));
}
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
index e8fbadfe7c5..63ce777d7fd 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
@@ -738,7 +738,7 @@ class SimpleTextFieldsReader extends FieldsProducer {
}
}
docCount = visitedDocs.cardinality();
- fst = fstCompiler.compile();
+ fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
/*
PrintStream ps = new PrintStream("out.dot");
fst.toDot(ps);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java
index 0a6c9010143..a73fef410dd 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/FSTDictionary.java
@@ -185,7 +185,8 @@ public class FSTDictionary implements IndexDictionary {
@Override
public FSTDictionary build() throws IOException {
- return new FSTDictionary(fstCompiler.compile());
+ return new FSTDictionary(
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 26aea1dda59..1cf045f9c14 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -587,7 +587,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
}
}
- index = fstCompiler.compile();
+ index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
assert subIndices == null;
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index 260c159f207..6bb5718d5c7 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -437,6 +437,21 @@ public final class FST<T> implements Accountable {
this.fstReader = fstReader;
}
+ /**
+ * Create a FST from a {@link FSTReader}. Return null if the metadata is null.
+ *
+ * @param fstMetadata the metadata
+ * @param fstReader the FSTReader
+ * @return the FST
+ */
+ public static <T> FST<T> fromFSTReader(FSTMetadata<T> fstMetadata, FSTReader fstReader) {
+ // FSTMetadata could be null if there is no node accepted by the FST
+ if (fstMetadata == null) {
+ return null;
+ }
+ return new FST<>(fstMetadata, Objects.requireNonNull(fstReader, "FSTReader cannot be null"));
+ }
+
/**
* Read the FST metadata from DataInput
*
@@ -516,9 +531,7 @@ public final class FST<T> implements Accountable {
}
/**
- * Save the FST to DataOutput. If you use an {@link org.apache.lucene.store.IndexOutput} to build
- * the FST, then you should not and do not need to call this method, as the FST is already saved.
- * Doing so will throw an {@link UnsupportedOperationException}.
+ * Save the FST to DataOutput.
*
* @param metaOut the DataOutput to write the metadata to
* @param out the DataOutput to write the FST bytes to
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
index eaf26bb91eb..3f0b2b4deb2 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -99,6 +99,7 @@ public class FSTCompiler<T> {
private static final FSTReader NULL_FST_READER = new NullFSTReader();
private final NodeHash<T> dedupHash;
+ // a temporary FST used during building for NodeHash cache
final FST<T> fst;
private final T NO_OUTPUT;
@@ -173,9 +174,7 @@ public class FSTCompiler<T> {
paddingBytePending = true;
this.dataOutput = dataOutput;
fst =
- new FST<>(
- new FST.FSTMetadata<>(inputType, outputs, null, -1, version, 0),
- toFSTReader(dataOutput));
+ new FST<>(new FST.FSTMetadata<>(inputType, outputs, null, -1, version, 0), NULL_FST_READER);
if (suffixRAMLimitMB < 0) {
throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
} else if (suffixRAMLimitMB > 0) {
@@ -193,16 +192,6 @@ public class FSTCompiler<T> {
}
}
- // Get the respective FSTReader of the DataOutput. If the DataOutput is also a FSTReader then we
- // will use it, otherwise we will return a NullFSTReader. Attempting to read from a FST with
- // NullFSTReader will throw UnsupportedOperationException
- private FSTReader toFSTReader(DataOutput dataOutput) {
- if (dataOutput instanceof FSTReader) {
- return (FSTReader) dataOutput;
- }
- return NULL_FST_READER;
- }
-
/**
* This class is used for FST backed by non-FSTReader DataOutput. It does not allow getting the
* reverse BytesReader nor writing to a DataOutput.
@@ -227,6 +216,22 @@ public class FSTCompiler<T> {
}
}
+ /**
+ * Get the respective {@link FSTReader} of the {@link DataOutput}. To call this method, you need
+ * to use the default DataOutput or {@link #getOnHeapReaderWriter(int)}, otherwise we will throw
+ * an exception.
+ *
+ * @return the DataOutput as FSTReader
+ * @throws IllegalStateException if the DataOutput does not implement FSTReader
+ */
+ public FSTReader getFSTReader() {
+ if (dataOutput instanceof FSTReader) {
+ return (FSTReader) dataOutput;
+ }
+ throw new IllegalStateException(
+ "The DataOutput must implement FSTReader, but got " + dataOutput);
+ }
+
/**
* Fluent-style constructor for FST {@link FSTCompiler}.
*
@@ -968,10 +973,31 @@ public class FSTCompiler<T> {
return output == NO_OUTPUT || !output.equals(NO_OUTPUT);
}
- /** Returns final FST. NOTE: this will return null if nothing is accepted by the FST. */
- // TODO: make this method to only return the FSTMetadata and user needs to construct the FST
- // themselves
- public FST<T> compile() throws IOException {
+ /**
+ * Returns the metadata of the final FST. NOTE: this will return null if nothing is accepted by
+ * the FST themselves.
+ *
+ * <p>To create the FST, you need to:
+ *
+ * <p>- If a FSTReader DataOutput was used, such as the one returned by {@link
+ * #getOnHeapReaderWriter(int)}
+ *
+ * <pre class="prettyprint">
+ * fstMetadata = fstCompiler.compile();
+ * fst = FST.fromFSTReader(fstMetadata, fstCompiler.getFSTReader());
+ * </pre>
+ *
+ * <p>- If a non-FSTReader DataOutput was used, such as {@link
+ * org.apache.lucene.store.IndexOutput}, you need to first create the corresponding {@link
+ * org.apache.lucene.store.DataInput}, such as {@link org.apache.lucene.store.IndexInput} then
+ * pass it to the FST construct
+ *
+ * <pre class="prettyprint">
+ * fstMetadata = fstCompiler.compile();
+ * fst = new FST<>(fstMetadata, dataInput, new OffHeapFSTStore());
+ * </pre>
+ */
+ public FST.FSTMetadata<T> compile() throws IOException {
final UnCompiledNode<T> root = frontier[0];
@@ -991,7 +1017,7 @@ public class FSTCompiler<T> {
// root.output=" + root.output);
finish(compileNode(root).node);
- return fst;
+ return fst.metadata;
}
/** Expert: holds a pending (seen but not yet serialized) arc. */
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/package-info.java b/lucene/core/src/java/org/apache/lucene/util/fst/package-info.java
index aa7bd91f8ec..1f80869f699 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/package-info.java
@@ -49,7 +49,7 @@
* scratchBytes.copyChars(inputValues[i]);
* fstCompiler.add(Util.toIntsRef(scratchBytes.toBytesRef(), scratchInts), outputValues[i]);
* }
- * FST<Long> fst = fstCompiler.compile();
+ * FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
* </pre>
*
* Retrieval by key:
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
index b2758ca526e..1cf72208a93 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFST.java
@@ -90,7 +90,7 @@ public class Test2BFST extends LuceneTestCase {
nextInput(r, ints2);
}
- FST<Object> fst = fstCompiler.compile();
+ FST<Object> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
for (int verify = 0; verify < 2; verify++) {
System.out.println(
@@ -183,7 +183,7 @@ public class Test2BFST extends LuceneTestCase {
nextInput(r, ints);
}
- FST<BytesRef> fst = fstCompiler.compile();
+ FST<BytesRef> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
for (int verify = 0; verify < 2; verify++) {
System.out.println(
@@ -273,7 +273,7 @@ public class Test2BFST extends LuceneTestCase {
nextInput(r, ints);
}
- FST<Long> fst = fstCompiler.compile();
+ FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
for (int verify = 0; verify < 2; verify++) {
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
index 090c99716f6..1773c47f432 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/Test2BFSTOffHeap.java
@@ -92,10 +92,10 @@ public class Test2BFSTOffHeap extends LuceneTestCase {
nextInput(r, ints2);
}
- FST<Object> fst = fstCompiler.compile();
+ FST.FSTMetadata<Object> fstMetadata = fstCompiler.compile();
indexOutput.close();
try (IndexInput indexInput = dir.openInput("fst", IOContext.DEFAULT)) {
- fst = new FST<>(fst.getMetadata(), indexInput, new OffHeapFSTStore());
+ FST<Object> fst = new FST<>(fstMetadata, indexInput, new OffHeapFSTStore());
for (int verify = 0; verify < 2; verify++) {
System.out.println(
@@ -180,10 +180,10 @@ public class Test2BFSTOffHeap extends LuceneTestCase {
nextInput(r, ints);
}
- FST<BytesRef> fst = fstCompiler.compile();
+ FST.FSTMetadata<BytesRef> fstMetadata = fstCompiler.compile();
indexOutput.close();
try (IndexInput indexInput = dir.openInput("fst", IOContext.DEFAULT)) {
- fst = new FST<>(fst.getMetadata(), indexInput, new OffHeapFSTStore());
+ FST<BytesRef> fst = new FST<>(fstMetadata, indexInput, new OffHeapFSTStore());
for (int verify = 0; verify < 2; verify++) {
System.out.println(
@@ -265,10 +265,10 @@ public class Test2BFSTOffHeap extends LuceneTestCase {
nextInput(r, ints);
}
- FST<Long> fst = fstCompiler.compile();
+ FST.FSTMetadata<Long> fstMetadata = fstCompiler.compile();
indexOutput.close();
try (IndexInput indexInput = dir.openInput("fst", IOContext.DEFAULT)) {
- fst = new FST<>(fst.getMetadata(), indexInput, new OffHeapFSTStore());
+ FST<Long> fst = new FST<>(fstMetadata, indexInput, new OffHeapFSTStore());
for (int verify = 0; verify < 2; verify++) {
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
index 0f064d1ea44..81792f8ed75 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTDirectAddressing.java
@@ -196,7 +196,7 @@ public class TestFSTDirectAddressing extends LuceneTestCase {
}
last = entry;
}
- return fstCompiler.compile();
+ return FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
public static void main(String... args) throws Exception {
@@ -333,7 +333,7 @@ public class TestFSTDirectAddressing extends LuceneTestCase {
while ((inputOutput = fstEnum.next()) != null) {
fstCompiler.add(inputOutput.input, CharsRef.deepCopyOf(inputOutput.output));
}
- return fstCompiler.compile();
+ return FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
private static int walk(FST<CharsRef> read) throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
index 7a4e9f09980..20339879cf0 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
@@ -407,7 +407,7 @@ public class TestFSTs extends LuceneTestCase {
System.out.println(ord + " terms...");
}
}
- FST<Long> fst = fstCompiler.compile();
+ FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
if (VERBOSE) {
System.out.println(
"FST: "
@@ -569,7 +569,7 @@ public class TestFSTs extends LuceneTestCase {
System.out.println(
((tMid - tStart) / (double) TimeUnit.SECONDS.toNanos(1)) + " sec to add all terms");
- FST<T> fst = fstCompiler.compile();
+ FST<T> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
long tEnd = System.nanoTime();
System.out.println(
((tEnd - tMid) / (double) TimeUnit.SECONDS.toNanos(1)) + " sec to finish/pack");
@@ -774,7 +774,8 @@ public class TestFSTs extends LuceneTestCase {
new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();
fstCompiler.add(
Util.toIntsRef(newBytesRef("foobar"), new IntsRefBuilder()), outputs.getNoOutput());
- final BytesRefFSTEnum<Object> fstEnum = new BytesRefFSTEnum<>(fstCompiler.compile());
+ final BytesRefFSTEnum<Object> fstEnum =
+ new BytesRefFSTEnum<>(FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()));
assertNull(fstEnum.seekFloor(newBytesRef("foo")));
assertNull(fstEnum.seekCeil(newBytesRef("foobaz")));
}
@@ -788,7 +789,7 @@ public class TestFSTs extends LuceneTestCase {
for (int i = 0; i < 10; i++) {
fstCompiler.add(Util.toIntsRef(newBytesRef(str), ints), outputs.getNoOutput());
}
- FST<Object> fst = fstCompiler.compile();
+ FST<Object> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// count the input paths
int count = 0;
@@ -863,7 +864,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(b, new IntsRefBuilder()), 42L);
fstCompiler.add(Util.toIntsRef(c, new IntsRefBuilder()), 13824324872317238L);
- final FST<Long> fst = fstCompiler.compile();
+ final FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
assertEquals(13824324872317238L, (long) Util.get(fst, c));
assertEquals(42, (long) Util.get(fst, b));
@@ -1107,7 +1108,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(term.get(), scratchIntsRef), nothing);
}
- return fstCompiler.compile();
+ return FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
void generate(ArrayList<String> out, StringBuilder b, char from, char to, int depth) {
@@ -1173,7 +1174,7 @@ public class TestFSTs extends LuceneTestCase {
new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE4, outputs).build();
fstCompiler.add(Util.toUTF32("slat", new IntsRefBuilder()), 10L);
fstCompiler.add(Util.toUTF32("st", new IntsRefBuilder()), 17L);
- final FST<Long> fst = fstCompiler.compile();
+ final FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));
StringWriter w = new StringWriter();
Util.toDot(fst, w, false, false);
@@ -1190,7 +1191,7 @@ public class TestFSTs extends LuceneTestCase {
Util.toIntsRef(newBytesRef("stat"), new IntsRefBuilder()), outputs.getNoOutput());
fstCompiler.add(
Util.toIntsRef(newBytesRef("station"), new IntsRefBuilder()), outputs.getNoOutput());
- final FST<Long> fst = fstCompiler.compile();
+ final FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
StringWriter w = new StringWriter();
// Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp/out.dot"));
Util.toDot(fst, w, false, false);
@@ -1216,7 +1217,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(newBytesRef("aac"), scratch), 7L);
fstCompiler.add(Util.toIntsRef(newBytesRef("ax"), scratch), 17L);
- FST<Long> fst = fstCompiler.compile();
+ FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// save the FST to DataOutput, here it would not matter whether we are saving to different
// DataOutput for meta or not
@@ -1252,7 +1253,6 @@ public class TestFSTs extends LuceneTestCase {
final Long nothing = outputs.getNoOutput();
final FSTCompiler<Long> fstCompiler =
new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).build();
- final FST<Long> fst = fstCompiler.fst;
final FSTCompiler.UnCompiledNode<Long> rootNode =
new FSTCompiler.UnCompiledNode<>(fstCompiler, 0);
@@ -1285,6 +1285,8 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.finish(fstCompiler.addNode(rootNode));
+ final FST<Long> fst = new FST<>(fstCompiler.fst.metadata, fstCompiler.getFSTReader());
+
StringWriter w = new StringWriter();
// Writer w = new OutputStreamWriter(new FileOutputStream("/x/tmp3/out.dot"));
Util.toDot(fst, w, false, false);
@@ -1333,7 +1335,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(newBytesRef("aab"), scratch), 22L);
fstCompiler.add(Util.toIntsRef(newBytesRef("aac"), scratch), 7L);
fstCompiler.add(Util.toIntsRef(newBytesRef("ax"), scratch), 17L);
- final FST<Long> fst = fstCompiler.compile();
+ final FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
// Util.toDot(fst, w, false, false);
// w.close();
@@ -1370,7 +1372,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(newBytesRef("adcde"), scratch), 17L);
fstCompiler.add(Util.toIntsRef(newBytesRef("ax"), scratch), 17L);
- final FST<Long> fst = fstCompiler.compile();
+ final FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
final AtomicInteger rejectCount = new AtomicInteger();
Util.TopNSearcher<Long> searcher =
new Util.TopNSearcher<>(fst, 2, 6, minLongComparator) {
@@ -1433,7 +1435,8 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(newBytesRef("aab"), scratch), outputs.newPair(22L, 57L));
fstCompiler.add(Util.toIntsRef(newBytesRef("aac"), scratch), outputs.newPair(7L, 36L));
fstCompiler.add(Util.toIntsRef(newBytesRef("ax"), scratch), outputs.newPair(17L, 85L));
- final FST<Pair<Long, Long>> fst = fstCompiler.compile();
+ final FST<Pair<Long, Long>> fst =
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
// Util.toDot(fst, w, false, false);
// w.close();
@@ -1492,7 +1495,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(newBytesRef(e.getKey()), scratch), e.getValue());
}
- final FST<Long> fst = fstCompiler.compile();
+ final FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// System.out.println("SAVE out.dot");
// Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
// Util.toDot(fst, w, false, false);
@@ -1619,7 +1622,8 @@ public class TestFSTs extends LuceneTestCase {
Util.toIntsRef(newBytesRef(e.getKey()), scratch), outputs.newPair(weight, output));
}
- final FST<Pair<Long, Long>> fst = fstCompiler.compile();
+ final FST<Pair<Long, Long>> fst =
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
// System.out.println("SAVE out.dot");
// Writer w = new OutputStreamWriter(new FileOutputStream("out.dot"));
// Util.toDot(fst, w, false, false);
@@ -1695,7 +1699,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(input.get(), newBytesRef(BytesRef.deepCopyOf(output)));
}
- final FST<BytesRef> fst = fstCompiler.compile();
+ final FST<BytesRef> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
for (int arc = 0; arc < 6; arc++) {
input.setIntAt(0, arc);
final BytesRef result = Util.get(fst, input.get());
@@ -1737,7 +1741,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(input.get(), term);
}
- FST<BytesRef> fst = fstCompiler.compile();
+ FST<BytesRef> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
Arc<BytesRef> arc = new FST.Arc<>();
fst.getFirstArc(arc);
@@ -1772,7 +1776,7 @@ public class TestFSTs extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(ac, new IntsRefBuilder()), 5L);
fstCompiler.add(Util.toIntsRef(bd, new IntsRefBuilder()), 7L);
- FST<Long> fst = fstCompiler.compile();
+ FST<Long> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
assertEquals(3, (long) Util.get(fst, ab));
assertEquals(5, (long) Util.get(fst, ac));
diff --git a/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java b/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java
index 1e6c34e7d8e..cbb37cc221c 100644
--- a/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/util/fst/TestUtil.java
@@ -116,6 +116,6 @@ public class TestUtil extends LuceneTestCase {
fstCompiler.add(
Util.toIntsRef(new BytesRef(word), new IntsRefBuilder()), outputs.getNoOutput());
}
- return fstCompiler.compile();
+ return FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
}
diff --git a/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java b/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java
index bb65e8b63cf..30c813e5c5a 100644
--- a/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java
+++ b/lucene/demo/src/java/org/apache/lucene/demo/knn/KnnVectorDict.java
@@ -155,7 +155,7 @@ public class KnnVectorDict implements Closeable {
while (addOneLine(in, binOut)) {
// continue;
}
- fstCompiler.compile().save(fstOut, fstOut);
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader()).save(fstOut, fstOut);
binOut.writeInt(numFields - 1);
}
}
diff --git a/lucene/misc/src/test/org/apache/lucene/misc/util/fst/TestFSTsMisc.java b/lucene/misc/src/test/org/apache/lucene/misc/util/fst/TestFSTsMisc.java
index 2e991117ba1..44c342b29ab 100644
--- a/lucene/misc/src/test/org/apache/lucene/misc/util/fst/TestFSTsMisc.java
+++ b/lucene/misc/src/test/org/apache/lucene/misc/util/fst/TestFSTsMisc.java
@@ -174,7 +174,7 @@ public class TestFSTsMisc extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(new BytesRef("a"), scratch), 3L);
fstCompiler.add(Util.toIntsRef(new BytesRef("a"), scratch), 0L);
fstCompiler.add(Util.toIntsRef(new BytesRef("b"), scratch), 17L);
- final FST<Object> fst = fstCompiler.compile();
+ final FST<Object> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
Object output = Util.get(fst, new BytesRef("a"));
assertNotNull(output);
@@ -208,7 +208,7 @@ public class TestFSTsMisc extends LuceneTestCase {
fstCompiler.add(Util.toIntsRef(new BytesRef("a"), scratch), 0L);
fstCompiler.add(Util.toIntsRef(new BytesRef("b"), scratch), 0L);
- final FST<Object> fst = fstCompiler.compile();
+ final FST<Object> fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
Object output = Util.get(fst, new BytesRef(""));
assertNotNull(output);
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java
index 1443434d2ca..0e9c2332558 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java
@@ -407,7 +407,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
}
}
- index = fstCompiler.compile();
+ index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
assert subIndices == null;
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
index ffbb5c3772b..bc02c197522 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
@@ -587,7 +587,7 @@ public class AnalyzingSuggester extends Lookup {
fstCompiler.add(scratchInts.get(), outputs.newPair(cost, br));
}
}
- fst = fstCompiler.compile();
+ fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
count = newCount;
// Util.dotToFile(fst, "/tmp/suggest.dot");
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
index 3d45cd1f71b..87346e9482e 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
@@ -323,7 +323,7 @@ public class FreeTextSuggester extends Lookup {
fstCompiler.add(Util.toIntsRef(term, scratchInts), encodeWeight(termsEnum.totalTermFreq()));
}
- final FST<Long> newFst = fstCompiler.compile();
+ final FST<Long> newFst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
if (newFst == null) {
throw new IllegalArgumentException("need at least one suggestion");
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java
index f4daf2a36be..02d9f84d3d4 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/NRTSuggesterBuilder.java
@@ -104,7 +104,8 @@ final class NRTSuggesterBuilder {
* CompletionPostingsFormat.FSTLoadMode)})}
*/
public boolean store(DataOutput output) throws IOException {
- final FST<PairOutputs.Pair<Long, BytesRef>> fst = fstCompiler.compile();
+ final FST<PairOutputs.Pair<Long, BytesRef>> fst =
+ FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
if (fst == null) {
return false;
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java
index 29949f845a0..4f71b6e6c7b 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionBuilder.java
@@ -220,6 +220,6 @@ public class FSTCompletionBuilder {
}
}
- return count == 0 ? null : fstCompiler.compile();
+ return count == 0 ? null : FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
}
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
index 808b4d7edd6..a3c0fdd137f 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
@@ -126,7 +126,7 @@ public class WFSTCompletionLookup extends Lookup {
previous.copyBytes(scratch);
newCount++;
}
- fst = fstCompiler.compile();
+ fst = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
count = newCount;
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java
index 1564b2a7e88..521e8cbe645 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/fst/FSTTester.java
@@ -282,28 +282,33 @@ public class FSTTester<T> {
fstCompiler.add(pair.input, pair.output);
}
}
- FST<T> fst = fstCompiler.compile();
+
+ FST<T> fst = null;
+ FST.FSTMetadata<T> fstMetadata = fstCompiler.compile();
if (useOffHeap) {
indexOutput.close();
- if (fst == null) {
+ if (fstMetadata == null) {
dir.deleteFile("fstOffHeap.bin");
} else {
try (IndexInput in = dir.openInput("fstOffHeap.bin", IOContext.DEFAULT)) {
- fst = new FST<>(fst.getMetadata(), in);
+ fst = new FST<>(fstMetadata, in);
} finally {
dir.deleteFile("fstOffHeap.bin");
}
}
- } else if (random.nextBoolean() && fst != null) {
- IOContext context = LuceneTestCase.newIOContext(random);
- try (IndexOutput out = dir.createOutput("fst.bin", context)) {
- fst.save(out, out);
- }
- try (IndexInput in = dir.openInput("fst.bin", context)) {
- fst = new FST<>(FST.readMetadata(in, outputs), in);
- } finally {
- dir.deleteFile("fst.bin");
+ } else if (fstMetadata != null) {
+ fst = FST.fromFSTReader(fstMetadata, fstCompiler.getFSTReader());
+ if (random.nextBoolean()) {
+ IOContext context = LuceneTestCase.newIOContext(random);
+ try (IndexOutput out = dir.createOutput("fst.bin", context)) {
+ fst.save(out, out);
+ }
+ try (IndexInput in = dir.openInput("fst.bin", context)) {
+ fst = new FST<>(FST.readMetadata(in, outputs), in);
+ } finally {
+ dir.deleteFile("fst.bin");
+ }
}
}