You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2025/05/09 05:26:40 UTC
(lucene) branch branch_10x updated: A specialized Trie for Block Tree Index (#14563)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 65fd8648eb6 A specialized Trie for Block Tree Index (#14563)
65fd8648eb6 is described below
commit 65fd8648eb633b90ca7f7b69bbae39b373ddaa16
Author: Guo Feng <52...@users.noreply.github.com>
AuthorDate: Fri May 9 13:26:34 2025 +0800
A specialized Trie for Block Tree Index (#14563)
* A specialized Trie for Block Tree Index (#14333)
* Fix OOM of TestTrie (#14488)
* Compute the doc range more efficiently when flushing doc block (#14447)
* Fix TestForTooMuchCloning (#14547)
* Fix tests: too many calls to IndexInput.clone during merging (#14595)
* Simplify rootCodeFp (#14596)
---------
Co-authored-by: panguixin <pa...@bytedance.com>
---
gradle/generation/forUtil.gradle | 54 +-
lucene/CHANGES.txt | 4 +
.../checksums/generateForDeltaUtil101.json | 4 +
.../generated/checksums/generateForUtil101.json | 4 +
lucene/backward-codecs/src/java/module-info.java | 8 +-
.../backward_codecs}/lucene101/ForDeltaUtil.java | 5 +-
.../lucene/backward_codecs}/lucene101/ForUtil.java | 5 +-
.../backward_codecs}/lucene101/Lucene101Codec.java | 4 +-
.../lucene101/Lucene101PostingsFormat.java | 62 +-
.../lucene101/Lucene101PostingsReader.java | 41 +-
.../backward_codecs}/lucene101/PForUtil.java | 3 +-
.../lucene101/PostingDecodingUtil.java | 57 ++
.../backward_codecs}/lucene101/PostingsUtil.java | 2 +-
.../backward_codecs}/lucene101/gen_ForDeltaUtil.py | 5 +-
.../backward_codecs}/lucene101/gen_ForUtil.py | 5 +-
.../backward_codecs}/lucene101/package-info.java | 2 +-
.../lucene90/Lucene90PostingsFormat.java | 9 +-
.../lucene90/blocktree/CompressionAlgorithm.java | 2 +-
.../lucene90/blocktree/FieldReader.java | 8 +-
.../lucene90/blocktree/IntersectTermsEnum.java | 2 +-
.../blocktree/IntersectTermsEnumFrame.java | 2 +-
.../blocktree/Lucene90BlockTreeTermsReader.java | 4 +-
.../lucene90/blocktree/SegmentTermsEnum.java | 2 +-
.../lucene90/blocktree/SegmentTermsEnumFrame.java | 2 +-
.../backward_codecs}/lucene90/blocktree/Stats.java | 2 +-
.../lucene90/blocktree/package-info.java | 4 +-
.../lucene912/Lucene912PostingsFormat.java | 9 +-
.../lucene99/Lucene99PostingsFormat.java | 9 +-
.../services/org.apache.lucene.codecs.Codec | 1 +
.../org.apache.lucene.codecs.PostingsFormat | 1 +
.../lucene101/Lucene101PostingsWriter.java | 18 +-
.../Lucene101RWPostingsFormat.java} | 28 +-
.../lucene101/TestForDeltaUtil.java | 6 +-
.../backward_codecs}/lucene101/TestForUtil.java | 6 +-
.../lucene101/TestLucene101PostingsFormat.java | 10 +-
.../lucene101/TestLucene101PostingsFormatV0.java | 6 +-
.../backward_codecs}/lucene101/TestPForUtil.java | 6 +-
.../lucene101/TestPostingsUtil.java | 2 +-
.../lucene90/Lucene90RWPostingsFormat.java | 4 +-
.../lucene90/TestLucene90PostingsFormat.java | 4 +-
.../blocktree/Lucene90BlockTreeTermsWriter.java | 2 +-
.../lucene90/blocktree/TestMSBVLong.java | 2 +-
.../lucene912/Lucene912RWPostingsFormat.java | 2 +-
.../lucene912/TestLucene912PostingsFormat.java | 4 +-
.../lucene99/Lucene99RWPostingsFormat.java | 2 +-
.../lucene99/TestLucene99PostingsFormat.java | 4 +-
.../TestInt7HnswBackwardsCompatibility.java | 4 +-
.../benchmark/jmh/PostingIndexInputBenchmark.java | 6 +-
.../blocktreeords/BlockTreeOrdsPostingsFormat.java | 14 +-
.../blocktreeords/OrdsBlockTreeTermsWriter.java | 10 +-
.../lucene/codecs/memory/DirectPostingsFormat.java | 8 +-
.../lucene/codecs/memory/FSTPostingsFormat.java | 8 +-
.../uniformsplit/DeltaBaseTermStateSerializer.java | 14 +-
.../uniformsplit/UniformSplitPostingsFormat.java | 8 +-
.../lucene/codecs/uniformsplit/package-info.java | 2 +-
.../lucene90/tests/MockTermStateFactory.java | 2 +-
.../generated/checksums/generateForDeltaUtil.json | 4 +-
.../src/generated/checksums/generateForUtil.json | 4 +-
lucene/core/src/java/module-info.java | 8 +-
.../src/java/org/apache/lucene/codecs/Codec.java | 2 +-
.../apache/lucene/codecs/PostingsWriterBase.java | 4 +-
.../lucene/codecs/lucene102/package-info.java | 16 +-
.../{lucene101 => lucene103}/ForDeltaUtil.java | 4 +-
.../codecs/{lucene101 => lucene103}/ForUtil.java | 2 +-
.../Lucene103Codec.java} | 24 +-
.../Lucene103PostingsFormat.java} | 71 +--
.../Lucene103PostingsReader.java} | 100 ++--
.../Lucene103PostingsWriter.java} | 45 +-
.../codecs/{lucene101 => lucene103}/PForUtil.java | 2 +-
.../PostingIndexInput.java | 2 +-
.../{lucene101 => lucene103}/PostingsUtil.java | 2 +-
.../blocktree/CompressionAlgorithm.java | 2 +-
.../blocktree/FieldReader.java | 81 +--
.../blocktree/IntersectTermsEnum.java | 78 ++-
.../blocktree/IntersectTermsEnumFrame.java | 39 +-
.../blocktree/Lucene103BlockTreeTermsReader.java} | 33 +-
.../blocktree/Lucene103BlockTreeTermsWriter.java} | 189 ++----
.../blocktree/SegmentTermsEnum.java | 412 ++++---------
.../blocktree/SegmentTermsEnumFrame.java | 43 +-
.../{lucene90 => lucene103}/blocktree/Stats.java | 4 +-
.../codecs/lucene103/blocktree/TrieBuilder.java | 641 +++++++++++++++++++++
.../codecs/lucene103/blocktree/TrieReader.java | 228 ++++++++
.../blocktree/package-info.java | 6 +-
.../{lucene101 => lucene103}/gen_ForDeltaUtil.py | 4 +-
.../codecs/{lucene101 => lucene103}/gen_ForUtil.py | 2 +-
.../{lucene102 => lucene103}/package-info.java | 29 +-
.../vectorization/VectorizationProvider.java | 4 +-
.../java/org/apache/lucene/search/PhraseQuery.java | 10 +-
.../java/org/apache/lucene/search/ScorerUtil.java | 4 +-
.../services/org.apache.lucene.codecs.Codec | 2 +-
.../org.apache.lucene.codecs.PostingsFormat | 2 +-
.../{lucene101 => lucene103}/TestForDeltaUtil.java | 4 +-
.../{lucene101 => lucene103}/TestForUtil.java | 4 +-
.../TestLucene103PostingsFormat.java} | 24 +-
.../{lucene101 => lucene103}/TestPForUtil.java | 4 +-
.../{lucene101 => lucene103}/TestPostingsUtil.java | 2 +-
.../codecs/lucene103/blocktree/TestTrie.java | 239 ++++++++
...tLucene90StoredFieldsFormatHighCompression.java | 8 +-
.../apache/lucene/index/TestForTooMuchCloning.java | 19 +-
.../vectorization/TestPostingDecodingUtil.java | 2 +-
.../queries/intervals/TermIntervalsSource.java | 10 +-
.../codecs/idversion/IDVersionPostingsFormat.java | 8 +-
.../idversion/VersionBlockTreeTermsWriter.java | 10 +-
.../document/Completion101PostingsFormat.java | 4 +-
.../tests/codecs/blockterms/LuceneFixedGap.java | 12 +-
.../blockterms/LuceneVarGapDocFreqInterval.java | 12 +-
.../blockterms/LuceneVarGapFixedInterval.java | 12 +-
.../mockrandom/MockRandomPostingsFormat.java | 16 +-
.../UniformSplitRot13PostingsFormat.java | 8 +-
.../util/TestRuleSetupAndRestoreClassEnv.java | 4 +-
.../org/apache/lucene/tests/util/TestUtil.java | 10 +-
111 files changed, 1909 insertions(+), 1110 deletions(-)
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle
index b55fd0204fd..b73c69b6f12 100644
--- a/gradle/generation/forUtil.gradle
+++ b/gradle/generation/forUtil.gradle
@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
description "Regenerate gen_ForUtil.py"
group "generation"
- def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
+ def genDir = file("src/java/org/apache/lucene/codecs/lucene103")
def genScript = file("${genDir}/gen_ForUtil.py")
def genOutput = file("${genDir}/ForUtil.java")
@@ -48,7 +48,7 @@ configure(project(":lucene:core")) {
description "Regenerate gen_ForDeltaUtil.py"
group "generation"
- def genDir = file("src/java/org/apache/lucene/codecs/lucene101")
+ def genDir = file("src/java/org/apache/lucene/codecs/lucene103")
def genScript = file("${genDir}/gen_ForDeltaUtil.py")
def genOutput = file("${genDir}/ForDeltaUtil.java")
@@ -197,5 +197,55 @@ configure(project(":lucene:backward-codecs")) {
andThenTasks: ["spotlessJava", "spotlessJavaApply"],
mustRunBefore: [ "compileJava" ]
])
+
+ task generateForUtil101Internal() {
+ description "Regenerate gen_ForUtil.py"
+ group "generation"
+
+ def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene101")
+ def genScript = file("${genDir}/gen_ForUtil.py")
+ def genOutput = file("${genDir}/ForUtil.java")
+
+ inputs.file genScript
+ outputs.file genOutput
+
+ doLast {
+ quietExec {
+ workingDir genDir
+ executable project.externalTool("python3")
+ args = [ '-B', genScript ]
+ }
+ }
+ }
+
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil101Internal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
+
+ task generateForDeltaUtil101Internal() {
+ description "Regenerate gen_ForDeltaUtil.py"
+ group "generation"
+
+ def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene101")
+ def genScript = file("${genDir}/gen_ForDeltaUtil.py")
+ def genOutput = file("${genDir}/ForDeltaUtil.java")
+
+ inputs.file genScript
+ outputs.file genOutput
+
+ doLast {
+ quietExec {
+ workingDir genDir
+ executable project.externalTool("python3")
+ args = [ '-B', genScript ]
+ }
+ }
+ }
+
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForDeltaUtil101Internal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index af3babed439..2e7dcd12dd6 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -41,6 +41,10 @@ Optimizations
* GITHUB#14425: KeywordField.newSetQuery() reuses prefixed terms (Mikhail Khludnev)
+* GITHUB#14333: Introduce a specialized trie for block tree index, instead of FST. (Guo Feng)
+
+* GITHUB#14447: Compute the doc range more efficiently when flushing doc block. (Pan Guixin)
+
Bug Fixes
---------------------
(No changes)
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil101.json b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil101.json
new file mode 100644
index 00000000000..2cd3169551d
--- /dev/null
+++ b/lucene/backward-codecs/src/generated/checksums/generateForDeltaUtil101.json
@@ -0,0 +1,4 @@
+{
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForDeltaUtil.java": "f4cff08d9a5dd99f5332c2f9f6d386f0d7f58677",
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForDeltaUtil.py": "ea46cd6b2384fc1cddb8c1dc5e30bf5f76054d91"
+}
\ No newline at end of file
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil101.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil101.json
new file mode 100644
index 00000000000..99aee2265c9
--- /dev/null
+++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil101.json
@@ -0,0 +1,4 @@
+{
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForUtil.java": "5a7c2e1e09780a2ccd31c22a1e1fa47443cf2a32",
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForUtil.py": "c98cce3be0698048ebda6beaa9d404f25089930d"
+}
\ No newline at end of file
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 41057c95bbf..88cbd590951 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -31,6 +31,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs.lucene86;
exports org.apache.lucene.backward_codecs.lucene87;
exports org.apache.lucene.backward_codecs.lucene90;
+ exports org.apache.lucene.backward_codecs.lucene90.blocktree;
exports org.apache.lucene.backward_codecs.lucene91;
exports org.apache.lucene.backward_codecs.lucene92;
exports org.apache.lucene.backward_codecs.lucene94;
@@ -38,6 +39,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs.lucene99;
exports org.apache.lucene.backward_codecs.lucene912;
exports org.apache.lucene.backward_codecs.lucene100;
+ exports org.apache.lucene.backward_codecs.lucene101;
exports org.apache.lucene.backward_codecs.packed;
exports org.apache.lucene.backward_codecs.store;
@@ -48,7 +50,8 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat,
- org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat;
+ org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat,
+ org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
@@ -67,5 +70,6 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene95.Lucene95Codec,
org.apache.lucene.backward_codecs.lucene99.Lucene99Codec,
org.apache.lucene.backward_codecs.lucene912.Lucene912Codec,
- org.apache.lucene.backward_codecs.lucene100.Lucene100Codec;
+ org.apache.lucene.backward_codecs.lucene100.Lucene100Codec,
+ org.apache.lucene.backward_codecs.lucene101.Lucene101Codec;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForDeltaUtil.java
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForDeltaUtil.java
index ceec3ce3342..cced2474ec7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForDeltaUtil.java
@@ -16,12 +16,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
-import static org.apache.lucene.codecs.lucene101.ForUtil.*;
+import static org.apache.lucene.backward_codecs.lucene101.ForUtil.*;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.packed.PackedInts;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForUtil.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForUtil.java
index 5a466d0f8f6..61a58bf64d2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/ForUtil.java
@@ -16,10 +16,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
/**
@@ -29,7 +28,7 @@ import org.apache.lucene.store.DataOutput;
*/
public final class ForUtil {
- public static final int BLOCK_SIZE = 128;
+ static final int BLOCK_SIZE = 128;
static final int BLOCK_SIZE_LOG2 = 7;
static int expandMask16(int mask16) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101Codec.java
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101Codec.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101Codec.java
index 2b764b87685..2fde12e09ae 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101Codec.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.util.Objects;
import org.apache.lucene.codecs.Codec;
@@ -49,7 +49,7 @@ import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
*
* <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
*
- * @see org.apache.lucene.codecs.lucene101 package documentation for file format details.
+ * @see org.apache.lucene.backward_codecs.lucene101 package documentation for file format details.
* @lucene.experimental
*/
public class Lucene101Codec extends Codec {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsFormat.java
similarity index 91%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsFormat.java
index ae9964c0edc..5528532229f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsFormat.java
@@ -14,18 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
@@ -94,8 +92,8 @@ import org.apache.lucene.util.packed.PackedInts;
* <dd><b>Term Dictionary</b>
* <p>The .tim file contains the list of terms in each field along with per-term statistics
* (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on
- * the format.
+ * .doc, .pos, and .pay files. See Lucene90BlockTreeTermsWriter for more details on the
+ * format.
* <p>NOTE: The term dictionary can plug into different postings implementations: the postings
* writer/reader are actually responsible for encoding and decoding the PostingsHeader and
* TermMetadata sections described here:
@@ -146,7 +144,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <dl>
* <dd><b>Term Index</b>
* <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.
+ * randomly. See Lucene90BlockTreeTermsWriter for more details on the format.
* </dl>
*
* <a id="Frequencies"></a>
@@ -319,7 +317,7 @@ import org.apache.lucene.util.packed.PackedInts;
*
* @lucene.experimental
*/
-public final class Lucene101PostingsFormat extends PostingsFormat {
+public class Lucene101PostingsFormat extends PostingsFormat {
/** Filename extension for some small metadata about how postings are encoded. */
public static final String META_EXTENSION = "psm";
@@ -342,15 +340,13 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
/** Size of blocks. */
public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
- public static final int BLOCK_MASK = BLOCK_SIZE - 1;
-
/** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */
public static final int LEVEL1_FACTOR = 32;
/** Total number of docs covered by level 1 skip data: 32 * 128 = 4,096 */
public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;
- public static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
+ static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
/**
* Return the class that implements {@link ImpactsEnum} in this {@link PostingsFormat}. This is
@@ -379,55 +375,15 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
static final int VERSION_CURRENT = VERSION_DENSE_BLOCKS_AS_BITSETS;
- private final int version;
- private final int minTermBlockSize;
- private final int maxTermBlockSize;
-
/** Creates {@code Lucene101PostingsFormat} with default settings. */
public Lucene101PostingsFormat() {
- this(
- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
- }
-
- /**
- * Creates {@code Lucene101PostingsFormat} with custom values for {@code minBlockSize} and {@code
- * maxBlockSize} passed to block terms dictionary.
- *
- * @see
- * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
- */
- public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
- this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
- }
-
- /** Expert constructor that allows setting the version. */
- public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
super("Lucene101");
- if (version < VERSION_START || version > VERSION_CURRENT) {
- throw new IllegalArgumentException("Version out of range: " + version);
- }
- this.version = version;
- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
- this.minTermBlockSize = minTermBlockSize;
- this.maxTermBlockSize = maxTermBlockSize;
}
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state, version);
- boolean success = false;
- try {
- FieldsConsumer ret =
- new Lucene90BlockTreeTermsWriter(
- state, postingsWriter, minTermBlockSize, maxTermBlockSize);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
+ throw new UnsupportedOperationException(
+ "This postings format may not be used for writing, use the current postings format");
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsReader.java
similarity index 96%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsReader.java
index 9141ad5133d..d45a079ec32 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsReader.java
@@ -14,17 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
-
-import static org.apache.lucene.codecs.lucene101.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.LEVEL1_NUM_DOCS;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.META_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.VERSION_START;
+package org.apache.lucene.backward_codecs.lucene101;
+
+import static org.apache.lucene.backward_codecs.lucene101.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.LEVEL1_NUM_DOCS;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.META_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.VERSION_START;
import java.io.IOException;
import java.util.AbstractList;
@@ -32,10 +32,10 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.RandomAccess;
+import org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Impact;
import org.apache.lucene.index.Impacts;
@@ -44,8 +44,6 @@ import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.SegmentReadState;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
-import org.apache.lucene.internal.vectorization.VectorizationProvider;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.DataInput;
@@ -65,7 +63,6 @@ import org.apache.lucene.util.VectorUtil;
*/
public final class Lucene101PostingsReader extends PostingsReaderBase {
- static final VectorizationProvider VECTORIZATION_PROVIDER = VectorizationProvider.getInstance();
// Dummy impacts, composed of the maximum possible term frequency and the lowest possible
// (unsigned) norm value. This is typically used on tail blocks, which don't actually record
// impacts as the storage overhead would not be worth any query evaluation speedup, since there's
@@ -466,7 +463,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
if (needsPos) {
this.posIn = Lucene101PostingsReader.this.posIn.clone();
- posInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(posIn);
+ posInUtil = new PostingDecodingUtil(posIn);
posDeltaBuffer = new int[BLOCK_SIZE];
} else {
this.posIn = null;
@@ -476,7 +473,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
if (needsOffsets || needsPayloads) {
this.payIn = Lucene101PostingsReader.this.payIn.clone();
- payInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(payIn);
+ payInUtil = new PostingDecodingUtil(payIn);
} else {
this.payIn = null;
payInUtil = null;
@@ -519,7 +516,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
if (docIn == null) {
// lazy init
docIn = Lucene101PostingsReader.this.docIn.clone();
- docInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(docIn);
+ docInUtil = new PostingDecodingUtil(docIn);
}
prefetchPostings(docIn, termState);
}
@@ -1359,9 +1356,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
}
- /**
- * @see Lucene101PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)
- */
+ /** see Lucene101PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int) */
static int readVInt15(DataInput in) throws IOException {
short s = in.readShort();
if (s >= 0) {
@@ -1371,9 +1366,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
}
- /**
- * @see Lucene101PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)
- */
+ /** see Lucene101PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long) */
static long readVLong15(DataInput in) throws IOException {
short s = in.readShort();
if (s >= 0) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PForUtil.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/PForUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PForUtil.java
index fd8ecd056b4..b3d2685f98b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PForUtil.java
@@ -14,11 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.LongHeap;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PostingDecodingUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PostingDecodingUtil.java
new file mode 100644
index 00000000000..dcb980c066e
--- /dev/null
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PostingDecodingUtil.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.backward_codecs.lucene101;
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexInput;
+
+/** Utility class to decode postings. */
+class PostingDecodingUtil {
+
+ /** The wrapper {@link IndexInput}. */
+ final IndexInput in;
+
+ /** Sole constructor, called by sub-classes. */
+ PostingDecodingUtil(IndexInput in) {
+ this.in = in;
+ }
+
+ /**
+ * Core methods for decoding blocks of docs / freqs / positions / offsets.
+ *
+ * <ul>
+ * <li>Read {@code count} ints.
+ * <li>For all {@code i} >= 0 so that {@code bShift - i * dec} > 0, apply shift {@code
+ * bShift - i * dec} and store the result in {@code b} at offset {@code count * i}.
+ * <li>Apply mask {@code cMask} and store the result in {@code c} starting at offset {@code
+ * cIndex}.
+ * </ul>
+ */
+ void splitInts(int count, int[] b, int bShift, int dec, int bMask, int[] c, int cIndex, int cMask)
+ throws IOException {
+ // Default implementation, which takes advantage of the C2 compiler's loop unrolling and
+ // auto-vectorization.
+ in.readInts(c, cIndex, count);
+ int maxIter = (bShift - 1) / dec;
+ for (int i = 0; i < count; ++i) {
+ for (int j = 0; j <= maxIter; ++j) {
+ b[count * j + i] = (c[cIndex + i] >>> (bShift - j * dec)) & bMask;
+ }
+ c[cIndex + i] &= cMask;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingsUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PostingsUtil.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingsUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PostingsUtil.java
index 34431a3689f..1de3bc68d85 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingsUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/PostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForDeltaUtil.py
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForDeltaUtil.py
index b1b36db096a..1691a84e87c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForDeltaUtil.py
@@ -40,14 +40,13 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene101.ForUtil.*;
+import static org.apache.lucene.backward_codecs.lucene101.ForUtil.*;
/**
* Inspired from https://fulmicoton.com/posts/bitpacking/
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForUtil.py
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForUtil.py
index 4640c9bd886..a326a19f5ab 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/gen_ForUtil.py
@@ -40,10 +40,9 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
/**
@@ -55,7 +54,7 @@ import org.apache.lucene.store.DataOutput;
*/
public final class ForUtil {
- public static final int BLOCK_SIZE = 128;
+ static final int BLOCK_SIZE = 128;
static final int BLOCK_SIZE_LOG2 = 7;
static int expandMask16(int mask16) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/package-info.java
similarity index 94%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/package-info.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/package-info.java
index 8aa1c3b43a0..1ff928d08f1 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/package-info.java
@@ -16,4 +16,4 @@
*/
/** Lucene 10.1 file format. */
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java
index 0fceeb47296..ce6cfccc659 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsFormat.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
@@ -24,8 +25,6 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.MultiLevelSkipListWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -95,8 +94,8 @@ import org.apache.lucene.util.packed.PackedInts;
* <dd><b>Term Dictionary</b>
* <p>The .tim file contains the list of terms in each field along with per-term statistics
* (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on
- * the format.
+ * .doc, .pos, and .pay files. See Lucene90BlockTreeTermsWriter for more details on the
+ * format.
* <p>NOTE: The term dictionary can plug into different postings implementations: the postings
* writer/reader are actually responsible for encoding and decoding the PostingsHeader and
* TermMetadata sections described here:
@@ -150,7 +149,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <dl>
* <dd><b>Term Index</b>
* <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.
+ * randomly. See Lucene90BlockTreeTermsWriter for more details on the format.
* </dl>
*
* <a id="Frequencies"></a>
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/CompressionAlgorithm.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/CompressionAlgorithm.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/CompressionAlgorithm.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/CompressionAlgorithm.java
index de24dc3fb05..aba587cf499 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/CompressionAlgorithm.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/CompressionAlgorithm.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/FieldReader.java
similarity index 96%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/FieldReader.java
index 259639762ec..83286bb7695 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/FieldReader.java
@@ -14,9 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
-import static org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT;
+import static org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT;
import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
@@ -118,8 +118,8 @@ public final class FieldReader extends Terms {
}
/**
- * Decodes a variable length byte[] in MSB order back to long, as written by {@link
- * Lucene90BlockTreeTermsWriter#writeMSBVLong}.
+ * Decodes a variable length byte[] in MSB order back to long, as written by
+ * Lucene90BlockTreeTermsWriter#writeMSBVLong.
*
* <p>Package private for testing.
*/
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnum.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/IntersectTermsEnum.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnum.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/IntersectTermsEnum.java
index e2f284f780d..9b6cc8e4362 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnum.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/IntersectTermsEnum.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import org.apache.lucene.index.BaseTermsEnum;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
index 9f6bb75788e..ed69c95bfe6 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
index 3e2ddbbb5d5..fc28374275f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import java.util.ArrayList;
@@ -59,7 +59,7 @@ import org.apache.lucene.util.fst.Outputs;
* <p>Use {@link org.apache.lucene.index.CheckIndex} with the <code>-verbose</code> option to see
* summary statistics on the blocks in the dictionary.
*
- * <p>See {@link Lucene90BlockTreeTermsWriter}.
+ * <p>See Lucene90BlockTreeTermsWriter.
*
* @lucene.experimental
*/
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/SegmentTermsEnum.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/SegmentTermsEnum.java
index 45ec4ee06ba..ba6fe67b0f7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/SegmentTermsEnum.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import java.io.PrintStream;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
index 85d23a489fe..b7affa5c6f5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Stats.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Stats.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Stats.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Stats.java
index ceeef4a8687..8e0284050da 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Stats.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Stats.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import static java.nio.charset.StandardCharsets.UTF_8;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/package-info.java
similarity index 88%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/package-info.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/package-info.java
index 27d57b60371..c7b578896eb 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/package-info.java
@@ -23,7 +23,7 @@
* structure. It allows you to plug in your own {@link org.apache.lucene.codecs.PostingsWriterBase}
* to implement the postings.
*
- * <p>See {@link org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter} for the
+ * <p>See org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter for the
* file format.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/Lucene912PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/Lucene912PostingsFormat.java
index bb748f62495..695b6b0828b 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/Lucene912PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/Lucene912PostingsFormat.java
@@ -17,14 +17,13 @@
package org.apache.lucene.backward_codecs.lucene912;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -92,8 +91,8 @@ import org.apache.lucene.util.packed.PackedInts;
* <dd><b>Term Dictionary</b>
* <p>The .tim file contains the list of terms in each field along with per-term statistics
* (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on
- * the format.
+ * .doc, .pos, and .pay files. See Lucene90BlockTreeTermsWriter for more details on the
+ * format.
* <p>NOTE: The term dictionary can plug into different postings implementations: the postings
* writer/reader are actually responsible for encoding and decoding the PostingsHeader and
* TermMetadata sections described here:
@@ -144,7 +143,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <dl>
* <dd><b>Term Index</b>
* <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.
+ * randomly. See Lucene90BlockTreeTermsWriter for more details on the format.
* </dl>
*
* <a id="Frequencies"></a>
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java
index 7ff614d684e..f4fbc49c514 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java
@@ -17,6 +17,7 @@
package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
@@ -24,8 +25,6 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.MultiLevelSkipListWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -93,8 +92,8 @@ import org.apache.lucene.util.packed.PackedInts;
* <dd><b>Term Dictionary</b>
* <p>The .tim file contains the list of terms in each field along with per-term statistics
* (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on
- * the format.
+ * .doc, .pos, and .pay files. See Lucene90BlockTreeTermsWriter for more details on the
+ * format.
* <p>NOTE: The term dictionary can plug into different postings implementations: the postings
* writer/reader are actually responsible for encoding and decoding the PostingsHeader and
* TermMetadata sections described here:
@@ -148,7 +147,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <dl>
* <dd><b>Term Index</b>
* <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.
+ * randomly. See Lucene90BlockTreeTermsWriter for more details on the format.
* </dl>
*
* <a id="Frequencies"></a>
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index ff4d7eeda4e..9f6cf615a3e 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -25,3 +25,4 @@ org.apache.lucene.backward_codecs.lucene95.Lucene95Codec
org.apache.lucene.backward_codecs.lucene99.Lucene99Codec
org.apache.lucene.backward_codecs.lucene912.Lucene912Codec
org.apache.lucene.backward_codecs.lucene100.Lucene100Codec
+org.apache.lucene.backward_codecs.lucene101.Lucene101Codec
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 49d917dc427..84cf4182384 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -18,3 +18,4 @@ org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat
org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat
org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsFormat
+org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsWriter.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsWriter.java
index 3d19a69b82d..5ea23b201b0 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsWriter.java
@@ -14,25 +14,25 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.*;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.LEVEL1_MASK;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.META_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.*;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.LEVEL1_MASK;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.META_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.TERMS_CODEC;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Impact;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/Lucene101RWPostingsFormat.java
similarity index 67%
copy from lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/Lucene101RWPostingsFormat.java
index af1037432af..f9a2c6f219f 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/Lucene101RWPostingsFormat.java
@@ -14,37 +14,47 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.backward_codecs.lucene912;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
-/** Read-write impersonation of {@link Lucene912PostingsFormat}. */
-public final class Lucene912RWPostingsFormat extends Lucene912PostingsFormat {
+/** Read-write impersonation of {@link Lucene101PostingsFormat}. */
+public final class Lucene101RWPostingsFormat extends Lucene101PostingsFormat {
+ private final int version;
private final int minTermBlockSize;
private final int maxTermBlockSize;
- /** Creates {@code Lucene912PostingsFormat} with default settings. */
- public Lucene912RWPostingsFormat() {
+ /** Creates {@code Lucene101PostingsFormat} with default settings. */
+ public Lucene101RWPostingsFormat() {
this(
Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
}
/**
- * Creates {@code Lucene912PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * Creates {@code Lucene101PostingsFormat} with custom values for {@code minBlockSize} and {@code
* maxBlockSize} passed to block terms dictionary.
*
* @see
* Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
*/
- public Lucene912RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ public Lucene101RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
+ }
+
+ /** Expert constructor that allows setting the version. */
+ public Lucene101RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
super();
+ if (version < VERSION_START || version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version out of range: " + version);
+ }
+ this.version = version;
Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
this.minTermBlockSize = minTermBlockSize;
this.maxTermBlockSize = maxTermBlockSize;
@@ -52,7 +62,7 @@ public final class Lucene912RWPostingsFormat extends Lucene912PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state, version);
boolean success = false;
try {
FieldsConsumer ret =
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForDeltaUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestForDeltaUtil.java
similarity index 93%
copy from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForDeltaUtil.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestForDeltaUtil.java
index d41ab472ea6..4fe5ae4c300 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForDeltaUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestForDeltaUtil.java
@@ -14,12 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -67,8 +66,7 @@ public class TestForDeltaUtil extends LuceneTestCase {
{
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
- PostingDecodingUtil pdu =
- Lucene101PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);
ForDeltaUtil forDeltaUtil = new ForDeltaUtil();
for (int i = 0; i < iterations; ++i) {
int base = 0;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestForUtil.java
similarity index 93%
copy from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForUtil.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestForUtil.java
index d93cb0ab3ec..c2411e052a1 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestForUtil.java
@@ -14,12 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -70,8 +69,7 @@ public class TestForUtil extends LuceneTestCase {
{
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
- PostingDecodingUtil pdu =
- Lucene101PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);
ForUtil forUtil = new ForUtil();
for (int i = 0; i < iterations; ++i) {
final int bitsPerValue = in.readByte();
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestLucene101PostingsFormat.java
similarity index 94%
copy from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestLucene101PostingsFormat.java
index 6b0ff1fe5e3..91b9469a4af 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestLucene101PostingsFormat.java
@@ -14,17 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.lucene.backward_codecs.lucene101.Lucene101PostingsReader.MutableImpactList;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.FieldReader;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Stats;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader.MutableImpactList;
-import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Stats;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -45,7 +45,7 @@ public class TestLucene101PostingsFormat extends BasePostingsFormatTestCase {
@Override
protected Codec getCodec() {
- return TestUtil.alwaysPostingsFormat(new Lucene101PostingsFormat());
+ return TestUtil.alwaysPostingsFormat(new Lucene101RWPostingsFormat());
}
public void testVInt15() throws IOException {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormatV0.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestLucene101PostingsFormatV0.java
similarity index 87%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormatV0.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestLucene101PostingsFormatV0.java
index 037527413ea..7231dd2fa1a 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormatV0.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestLucene101PostingsFormatV0.java
@@ -14,10 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
@@ -26,7 +26,7 @@ public class TestLucene101PostingsFormatV0 extends BasePostingsFormatTestCase {
@Override
protected Codec getCodec() {
return TestUtil.alwaysPostingsFormat(
- new Lucene101PostingsFormat(
+ new Lucene101RWPostingsFormat(
Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE,
Lucene101PostingsFormat.VERSION_START));
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestPForUtil.java
similarity index 94%
copy from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPForUtil.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestPForUtil.java
index 4e9ab4b55ee..902c384fcc8 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPForUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestPForUtil.java
@@ -14,12 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
import java.util.Arrays;
-import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.ByteBuffersDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -40,8 +39,7 @@ public class TestPForUtil extends LuceneTestCase {
final long endPointer = encodeTestData(iterations, values, d);
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
- PostingDecodingUtil pdu =
- Lucene101PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ PostingDecodingUtil pdu = new PostingDecodingUtil(in);
final PForUtil pforUtil = new PForUtil();
for (int i = 0; i < iterations; ++i) {
if (random().nextInt(5) == 0) {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPostingsUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestPostingsUtil.java
similarity index 97%
copy from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPostingsUtil.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestPostingsUtil.java
index 5d02d0561e3..ef6587c1605 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPostingsUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene101/TestPostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.backward_codecs.lucene101;
import java.io.IOException;
import org.apache.lucene.store.Directory;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
index 4360b90f237..94f7cd57820 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
@@ -17,14 +17,14 @@
package org.apache.lucene.backward_codecs.lucene90;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.TermState;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
index 814285a8895..002e5bb2333 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
@@ -23,11 +23,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.lucene.backward_codecs.lucene90.Lucene90ScoreSkipReader.MutableImpactList;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.FieldReader;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Stats;
import org.apache.lucene.backward_codecs.lucene99.Lucene99SkipWriter;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Stats;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 8d0d5aaaa22..0c463dd2cd5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import static org.apache.lucene.util.fst.FSTCompiler.getOnHeapReaderWriter;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/blocktree/TestMSBVLong.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/blocktree/TestMSBVLong.java
similarity index 96%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene90/blocktree/TestMSBVLong.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/blocktree/TestMSBVLong.java
index 1ebab926209..c5bd7470290 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/blocktree/TestMSBVLong.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/blocktree/TestMSBVLong.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.backward_codecs.lucene90.blocktree;
import java.io.IOException;
import org.apache.lucene.store.ByteArrayDataInput;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java
index af1037432af..6728e6e1979 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/Lucene912RWPostingsFormat.java
@@ -17,9 +17,9 @@
package org.apache.lucene.backward_codecs.lucene912;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/TestLucene912PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/TestLucene912PostingsFormat.java
index adf8aaf9ec7..58eb4b33a38 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/TestLucene912PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene912/TestLucene912PostingsFormat.java
@@ -20,11 +20,11 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.FieldReader;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Stats;
import org.apache.lucene.backward_codecs.lucene912.Lucene912PostingsReader.MutableImpactList;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Stats;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java
index f513562cb35..a04a1a3c700 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java
@@ -17,9 +17,9 @@
package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java
index e954af88c3b..0d601d91a4c 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java
@@ -22,11 +22,11 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.FieldReader;
+import org.apache.lucene.backward_codecs.lucene90.blocktree.Stats;
import org.apache.lucene.backward_codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Stats;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
index 554591709a2..c5f1c63f6be 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt7HnswBackwardsCompatibility.java
@@ -23,7 +23,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsReader;
@@ -69,7 +69,7 @@ public class TestInt7HnswBackwardsCompatibility extends BackwardsCompatibilityTe
}
protected Codec getCodec() {
- return new Lucene101Codec() {
+ return new Lucene103Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new Lucene99HnswScalarQuantizedVectorsFormat(
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java
index 241b289c5f6..0881d0f6bf9 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/PostingIndexInputBenchmark.java
@@ -21,9 +21,9 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Random;
import java.util.concurrent.TimeUnit;
-import org.apache.lucene.codecs.lucene101.ForDeltaUtil;
-import org.apache.lucene.codecs.lucene101.ForUtil;
-import org.apache.lucene.codecs.lucene101.PostingIndexInput;
+import org.apache.lucene.codecs.lucene103.ForDeltaUtil;
+import org.apache.lucene.codecs.lucene103.ForUtil;
+import org.apache.lucene.codecs.lucene103.PostingIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
index 29854aa500f..68c842c72e9 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
@@ -22,14 +22,14 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
-/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene101PostingsWriter}. */
+/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene103PostingsWriter}. */
public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
private final int minTermBlockSize;
@@ -57,7 +57,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
super("BlockTreeOrds");
this.minTermBlockSize = minTermBlockSize;
this.maxTermBlockSize = maxTermBlockSize;
- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
}
@Override
@@ -67,7 +67,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
boolean success = false;
try {
@@ -84,7 +84,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
index 852c06ae5de..8518afabab8 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/OrdsBlockTreeTermsWriter.java
@@ -25,7 +25,7 @@ import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.blocktreeords.FSTOrdsOutputs.Output;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter; // javadocs
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter; // javadocs
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
@@ -80,9 +80,9 @@ import org.apache.lucene.util.fst.Util;
*/
/**
- * This is just like {@link Lucene90BlockTreeTermsWriter}, except it also stores a version per term,
- * and adds a method to its TermsEnum implementation to seekExact only if the version is >= the
- * specified version. The version is added to the terms index to avoid seeking if no term in the
+ * This is just like {@link Lucene103BlockTreeTermsWriter}, except it also stores a version per
+ * term, and adds a method to its TermsEnum implementation to seekExact only if the version is >=
+ * the specified version. The version is added to the terms index to avoid seeking if no term in the
* block has a high enough version. The term blocks file is .tiv and the terms index extension is
* .tipv.
*
@@ -170,7 +170,7 @@ public final class OrdsBlockTreeTermsWriter extends FieldsConsumer {
int minItemsInBlock,
int maxItemsInBlock)
throws IOException {
- Lucene90BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
+ Lucene103BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
maxDoc = state.segmentInfo.maxDoc();
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
index 5af23fb4945..5bb8b533d88 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
@@ -24,7 +24,7 @@ import java.util.TreeMap;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
@@ -54,7 +54,7 @@ import org.apache.lucene.util.automaton.TransitionAccessor;
// - or: longer dense skip lists than just next byte?
/**
- * Wraps {@link Lucene101PostingsFormat} format for on-disk storage, but then at read time loads and
+ * Wraps {@link Lucene103PostingsFormat} format for on-disk storage, but then at read time loads and
* stores all terms and postings directly in RAM as byte[], int[].
*
* <p><b>WARNING</b>: This is exceptionally RAM intensive: it makes no effort to compress the
@@ -97,12 +97,12 @@ public final class DirectPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return PostingsFormat.forName("Lucene101").fieldsConsumer(state);
+ return PostingsFormat.forName("Lucene103").fieldsConsumer(state);
}
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- FieldsProducer postings = PostingsFormat.forName("Lucene101").fieldsProducer(state);
+ FieldsProducer postings = PostingsFormat.forName("Lucene103").fieldsProducer(state);
if (state.context.context() != IOContext.Context.MERGE) {
FieldsProducer loadedPostings;
try {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
index 4893ee8ad26..224d47e8f91 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
@@ -22,8 +22,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -41,7 +41,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
boolean success = false;
try {
@@ -57,7 +57,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new FSTTermsReader(state, postingsReader);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
index 140d62b4e96..a4a9048a73e 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
@@ -17,13 +17,13 @@
package org.apache.lucene.codecs.uniformsplit;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.BLOCK_SIZE;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.TermState;
@@ -34,7 +34,7 @@ import org.apache.lucene.util.RamUsageEstimator;
/**
* {@link TermState} serializer which encodes each file pointer as a delta relative to a base file
- * pointer. It differs from {@link Lucene101PostingsWriter#encodeTerm} which encodes each file
+ * pointer. It differs from {@link Lucene103PostingsWriter#encodeTerm} which encodes each file
* pointer as a delta relative to the previous file pointer.
*
* <p>It automatically sets the base file pointer to the first valid file pointer for doc start FP,
@@ -95,7 +95,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Writes a {@link BlockTermState} to the provided {@link DataOutput}.
*
- * <p>Simpler variant of {@link Lucene101PostingsWriter#encodeTerm(DataOutput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene103PostingsWriter#encodeTerm(DataOutput, FieldInfo,
* BlockTermState, boolean)}.
*/
public void writeTermState(
@@ -145,7 +145,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Reads a {@link BlockTermState} from the provided {@link DataInput}.
*
- * <p>Simpler variant of {@link Lucene101PostingsReader#decodeTerm(DataInput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene103PostingsReader#decodeTerm(DataInput, FieldInfo,
* BlockTermState, boolean)}.
*
* @param reuse {@link BlockTermState} to reuse; or null to create a new one.
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
index 690eab21400..28c706332c3 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -113,7 +113,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
boolean success = false;
try {
FieldsConsumer termsWriter =
@@ -130,7 +130,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
boolean success = false;
try {
FieldsProducer termsReader =
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
index d31b28704ef..a4ac5603913 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
@@ -28,7 +28,7 @@
* org.apache.lucene.search.PhraseQuery})
* <li>Quite efficient for {@link org.apache.lucene.search.PrefixQuery}
* <li>Not efficient for spell-check and {@link org.apache.lucene.search.FuzzyQuery}, in this case
- * prefer {@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat}
+ * prefer {@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat}
* </ul>
*/
package org.apache.lucene.codecs.uniformsplit;
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
index 0708f3b8050..2f6dc6fa749 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
@@ -17,7 +17,7 @@
package org.apache.lucene.codecs.lucene90.tests;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
/** Test utility class to create mock {@link IntBlockTermState}. */
public class MockTermStateFactory {
diff --git a/lucene/core/src/generated/checksums/generateForDeltaUtil.json b/lucene/core/src/generated/checksums/generateForDeltaUtil.json
index 85765bbd7cb..539b766b84b 100644
--- a/lucene/core/src/generated/checksums/generateForDeltaUtil.json
+++ b/lucene/core/src/generated/checksums/generateForDeltaUtil.json
@@ -1,4 +1,4 @@
{
- "lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java": "87e4d19b5284fa39adf2c24328cae2076b6f7bb3",
- "lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py": "165586f801bef4d2f540521e81bc119880038b6c"
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java": "1d463b5044c3498d2372a06e3cd536917786fdd1",
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py": "722d50e56c0e805df0059e6ec89b30ca7d5f845a"
}
\ No newline at end of file
diff --git a/lucene/core/src/generated/checksums/generateForUtil.json b/lucene/core/src/generated/checksums/generateForUtil.json
index a1d4310d5db..55cf553dd17 100644
--- a/lucene/core/src/generated/checksums/generateForUtil.json
+++ b/lucene/core/src/generated/checksums/generateForUtil.json
@@ -1,4 +1,4 @@
{
- "lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java": "3004112150943413e0f7fcc3e56b74c4875c1d64",
- "lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py": "b1041b6b46caab789c04d99483ee016d550eeebc"
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java": "1ea2dac2a26be521a70cf74c37fe9134f7b42cb7",
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py": "0eb3ae0815b4b25e4d33074d96546e622e4278be"
}
\ No newline at end of file
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index 1faa5178e46..b9030b52a47 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -25,14 +25,14 @@ module org.apache.lucene.core {
exports org.apache.lucene.analysis.tokenattributes;
exports org.apache.lucene.analysis;
exports org.apache.lucene.codecs.compressing;
- exports org.apache.lucene.codecs.lucene90.blocktree;
exports org.apache.lucene.codecs.lucene90.compressing;
exports org.apache.lucene.codecs.lucene90;
exports org.apache.lucene.codecs.lucene94;
exports org.apache.lucene.codecs.lucene95;
exports org.apache.lucene.codecs.lucene99;
- exports org.apache.lucene.codecs.lucene101;
exports org.apache.lucene.codecs.lucene102;
+ exports org.apache.lucene.codecs.lucene103.blocktree;
+ exports org.apache.lucene.codecs.lucene103;
exports org.apache.lucene.codecs.perfield;
exports org.apache.lucene.codecs;
exports org.apache.lucene.document;
@@ -79,7 +79,7 @@ module org.apache.lucene.core {
provides org.apache.lucene.analysis.TokenizerFactory with
org.apache.lucene.analysis.standard.StandardTokenizerFactory;
provides org.apache.lucene.codecs.Codec with
- org.apache.lucene.codecs.lucene101.Lucene101Codec;
+ org.apache.lucene.codecs.lucene103.Lucene103Codec;
provides org.apache.lucene.codecs.DocValuesFormat with
org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
@@ -89,7 +89,7 @@ module org.apache.lucene.core {
org.apache.lucene.codecs.lucene102.Lucene102HnswBinaryQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene102.Lucene102BinaryQuantizedVectorsFormat;
provides org.apache.lucene.codecs.PostingsFormat with
- org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
+ org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
provides org.apache.lucene.index.SortFieldProvider with
org.apache.lucene.search.SortField.Provider,
org.apache.lucene.search.SortedNumericSortField.Provider,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
index 7be508c6fd1..026c3b7b36c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
@@ -56,7 +56,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
}
@SuppressWarnings("NonFinalStaticField")
- static Codec defaultCodec = LOADER.lookup("Lucene101");
+ static Codec defaultCodec = LOADER.lookup("Lucene103");
}
private final String name;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java b/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java
index cb1c874b0ea..1069c1e094f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/PostingsWriterBase.java
@@ -18,7 +18,7 @@ package org.apache.lucene.codecs;
import java.io.Closeable;
import java.io.IOException;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.TermsEnum;
@@ -28,7 +28,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
/**
- * Class that plugs into term dictionaries, such as {@link Lucene90BlockTreeTermsWriter}, and
+ * Class that plugs into term dictionaries, such as {@link Lucene103BlockTreeTermsWriter}, and
* handles writing postings.
*
* @see PostingsReaderBase
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
index 8f6fcb2ef5b..05fbb64cd83 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
@@ -151,15 +151,15 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term dictionary}. A
+ * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term dictionary}. A
* dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Frequency data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Frequency data}. For
* each term in the dictionary, the numbers of all the documents that contain that term, and
* the frequency of the term in that document, unless frequencies are omitted ({@link
* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Proximity data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Proximity data}. For
* each term in the dictionary, the positions that the term occurs in each document. Note that
* this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java
index ceec3ce3342..d66b3e33660 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForDeltaUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForDeltaUtil.java
@@ -16,9 +16,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
-import static org.apache.lucene.codecs.lucene101.ForUtil.*;
+import static org.apache.lucene.codecs.lucene103.ForUtil.*;
import java.io.IOException;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java
index 5a466d0f8f6..4ccfbd28125 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/ForUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/ForUtil.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java
similarity index 92%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101Codec.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java
index 2b764b87685..d2e3eea6ef9 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103Codec.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.util.Objects;
import org.apache.lucene.codecs.Codec;
@@ -45,14 +45,14 @@ import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
/**
- * Implements the Lucene 10.1 index format
+ * Implements the Lucene 10.3 index format
*
* <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
*
- * @see org.apache.lucene.codecs.lucene101 package documentation for file format details.
+ * @see org.apache.lucene.codecs.lucene103 package documentation for file format details.
* @lucene.experimental
*/
-public class Lucene101Codec extends Codec {
+public class Lucene103Codec extends Codec {
/** Configuration option for the codec. */
public enum Mode {
@@ -80,7 +80,7 @@ public class Lucene101Codec extends Codec {
new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
- return Lucene101Codec.this.getPostingsFormatForField(field);
+ return Lucene103Codec.this.getPostingsFormatForField(field);
}
};
@@ -89,7 +89,7 @@ public class Lucene101Codec extends Codec {
new PerFieldDocValuesFormat() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
- return Lucene101Codec.this.getDocValuesFormatForField(field);
+ return Lucene103Codec.this.getDocValuesFormatForField(field);
}
};
@@ -98,14 +98,14 @@ public class Lucene101Codec extends Codec {
new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return Lucene101Codec.this.getKnnVectorsFormatForField(field);
+ return Lucene103Codec.this.getKnnVectorsFormatForField(field);
}
};
private final StoredFieldsFormat storedFieldsFormat;
/** Instantiates a new codec. */
- public Lucene101Codec() {
+ public Lucene103Codec() {
this(Mode.BEST_SPEED);
}
@@ -114,11 +114,11 @@ public class Lucene101Codec extends Codec {
*
* @param mode stored fields compression mode to use for newly flushed/merged segments.
*/
- public Lucene101Codec(Mode mode) {
- super("Lucene101");
+ public Lucene103Codec(Mode mode) {
+ super("Lucene103");
this.storedFieldsFormat =
new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);
- this.defaultPostingsFormat = new Lucene101PostingsFormat();
+ this.defaultPostingsFormat = new Lucene103PostingsFormat();
this.defaultDVFormat = new Lucene90DocValuesFormat();
this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
}
@@ -171,7 +171,7 @@ public class Lucene101Codec extends Codec {
/**
* Returns the postings format that should be used for writing new segments of <code>field</code>.
*
- * <p>The default implementation always returns "Lucene101".
+ * <p>The default implementation always returns "Lucene103".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java
similarity index 91%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java
index ae9964c0edc..3c489f9d3ca 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
@@ -24,8 +24,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.SegmentReadState;
@@ -36,7 +36,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
/**
- * Lucene 10.1 postings format, which encodes postings in packed integer blocks for fast decode.
+ * Lucene 10.3 postings format, which encodes postings in packed integer blocks for fast decode.
*
* <p>Basic idea:
*
@@ -48,7 +48,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block
* size is variable.
* <li><b>Block structure</b>:
- * <p>When the postings are long enough, Lucene101PostingsFormat will try to encode most
+ * <p>When the postings are long enough, Lucene103PostingsFormat will try to encode most
* integer data as a packed block.
* <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as
* two packed blocks, while the remaining 3 are encoded as one VInt block.
@@ -94,7 +94,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <dd><b>Term Dictionary</b>
* <p>The .tim file contains the list of terms in each field along with per-term statistics
* (such as docfreq) and pointers to the frequencies, positions, payload and skip data in the
- * .doc, .pos, and .pay files. See {@link Lucene90BlockTreeTermsWriter} for more details on
+ * .doc, .pos, and .pay files. See {@link Lucene103BlockTreeTermsWriter} for more details on
* the format.
* <p>NOTE: The term dictionary can plug into different postings implementations: the postings
* writer/reader are actually responsible for encoding and decoding the PostingsHeader and
@@ -146,7 +146,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <dl>
* <dd><b>Term Index</b>
* <p>The .tip file contains an index into the term dictionary, so that it can be accessed
- * randomly. See {@link Lucene90BlockTreeTermsWriter} for more details on the format.
+ * randomly. See {@link Lucene103BlockTreeTermsWriter} for more details on the format.
* </dl>
*
* <a id="Frequencies"></a>
@@ -167,8 +167,8 @@ import org.apache.lucene.util.packed.PackedInts;
* <DocDelta[,Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
* <li>Level1SkipData --> DocDelta, DocFPDelta, Skip1NumBytes?, ImpactLength?, Impacts?,
* PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
- * <li>Level0SkipData --> Skip0NumBytes, DocDelta, DocFPDelta, ImpactLength?, Impacts?,
- * PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
+ * <li>Level0SkipData --> Skip0NumBytes, DocDelta, DocFPDelta, PackedBlockLength,
+ * ImpactLength?, Impacts?, PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
* <li>PackedFreqBlock --> {@link PackedInts PackedInts}, uses patching
* <li>PackedDocDeltaBlock --> {@link PackedInts PackedInts}, does not use patching
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}
@@ -319,7 +319,7 @@ import org.apache.lucene.util.packed.PackedInts;
*
* @lucene.experimental
*/
-public final class Lucene101PostingsFormat extends PostingsFormat {
+public final class Lucene103PostingsFormat extends PostingsFormat {
/** Filename extension for some small metadata about how postings are encoded. */
public static final String META_EXTENSION = "psm";
@@ -359,67 +359,60 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
* @lucene.internal
*/
public static Class<? extends ImpactsEnum> getImpactsEnumImpl() {
- return Lucene101PostingsReader.BlockPostingsEnum.class;
+ return Lucene103PostingsReader.BlockPostingsEnum.class;
}
- static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";
- static final String META_CODEC = "Lucene101PostingsWriterMeta";
- static final String DOC_CODEC = "Lucene101PostingsWriterDoc";
- static final String POS_CODEC = "Lucene101PostingsWriterPos";
- static final String PAY_CODEC = "Lucene101PostingsWriterPay";
+ static final String TERMS_CODEC = "Lucene103PostingsWriterTerms";
+ static final String META_CODEC = "Lucene103PostingsWriterMeta";
+ static final String DOC_CODEC = "Lucene103PostingsWriterDoc";
+ static final String POS_CODEC = "Lucene103PostingsWriterPos";
+ static final String PAY_CODEC = "Lucene103PostingsWriterPay";
static final int VERSION_START = 0;
- /**
- * Version that started encoding dense blocks as bit sets. Note: the old format is a subset of the
- * new format, so Lucene101PostingsReader is able to read the old format without checking the
- * version.
- */
- static final int VERSION_DENSE_BLOCKS_AS_BITSETS = 1;
-
- static final int VERSION_CURRENT = VERSION_DENSE_BLOCKS_AS_BITSETS;
+ static final int VERSION_CURRENT = VERSION_START;
private final int version;
private final int minTermBlockSize;
private final int maxTermBlockSize;
- /** Creates {@code Lucene101PostingsFormat} with default settings. */
- public Lucene101PostingsFormat() {
+ /** Creates {@code Lucene103PostingsFormat} with default settings. */
+ public Lucene103PostingsFormat() {
this(
- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ Lucene103BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene103BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
}
/**
- * Creates {@code Lucene101PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * Creates {@code Lucene103PostingsFormat} with custom values for {@code minBlockSize} and {@code
* maxBlockSize} passed to block terms dictionary.
*
* @see
- * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
+ * Lucene103BlockTreeTermsWriter#Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
*/
- public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ public Lucene103PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
this(minTermBlockSize, maxTermBlockSize, VERSION_CURRENT);
}
/** Expert constructor that allows setting the version. */
- public Lucene101PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
- super("Lucene101");
+ public Lucene103PostingsFormat(int minTermBlockSize, int maxTermBlockSize, int version) {
+ super("Lucene103");
if (version < VERSION_START || version > VERSION_CURRENT) {
throw new IllegalArgumentException("Version out of range: " + version);
}
this.version = version;
- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ Lucene103BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
this.minTermBlockSize = minTermBlockSize;
this.maxTermBlockSize = maxTermBlockSize;
}
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state, version);
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state, version);
boolean success = false;
try {
FieldsConsumer ret =
- new Lucene90BlockTreeTermsWriter(
+ new Lucene103BlockTreeTermsWriter(
state, postingsWriter, minTermBlockSize, maxTermBlockSize);
success = true;
return ret;
@@ -432,10 +425,10 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
boolean success = false;
try {
- FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);
+ FieldsProducer ret = new Lucene103BlockTreeTermsReader(postingsReader, state);
success = true;
return ret;
} finally {
@@ -446,7 +439,7 @@ public final class Lucene101PostingsFormat extends PostingsFormat {
}
/**
- * Holds all state required for {@link Lucene101PostingsReader} to produce a {@link
+ * Holds all state required for {@link Lucene103PostingsReader} to produce a {@link
* org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
*
* @lucene.internal
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
similarity index 93%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
index 9141ad5133d..ccacfc4658a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsReader.java
@@ -14,17 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
-
-import static org.apache.lucene.codecs.lucene101.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.LEVEL1_NUM_DOCS;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.META_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.VERSION_START;
+package org.apache.lucene.codecs.lucene103;
+
+import static org.apache.lucene.codecs.lucene103.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.LEVEL1_NUM_DOCS;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.META_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.VERSION_START;
import java.io.IOException;
import java.util.AbstractList;
@@ -35,7 +35,7 @@ import java.util.RandomAccess;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Impact;
import org.apache.lucene.index.Impacts;
@@ -63,7 +63,7 @@ import org.apache.lucene.util.VectorUtil;
*
* @lucene.experimental
*/
-public final class Lucene101PostingsReader extends PostingsReaderBase {
+public final class Lucene103PostingsReader extends PostingsReaderBase {
static final VectorizationProvider VECTORIZATION_PROVIDER = VectorizationProvider.getInstance();
// Dummy impacts, composed of the maximum possible term frequency and the lowest possible
@@ -73,10 +73,6 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
private static final List<Impact> DUMMY_IMPACTS =
Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
- // We stopped storing a placeholder impact with freq=1 for fields with DOCS after 9.12.0
- private static final List<Impact> DUMMY_IMPACTS_NO_FREQS =
- Collections.singletonList(new Impact(1, 1L));
-
private final IndexInput docIn;
private final IndexInput posIn;
private final IndexInput payIn;
@@ -87,10 +83,10 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
private final int maxImpactNumBytesAtLevel1;
/** Sole constructor. */
- public Lucene101PostingsReader(SegmentReadState state) throws IOException {
+ public Lucene103PostingsReader(SegmentReadState state) throws IOException {
String metaName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.META_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.META_EXTENSION);
final long expectedDocFileLength, expectedPosFileLength, expectedPayFileLength;
ChecksumIndexInput metaIn = null;
boolean success = false;
@@ -150,7 +146,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
String docName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.DOC_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.DOC_EXTENSION);
try {
// Postings have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
// readahead.
@@ -162,7 +158,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
if (state.fieldInfos.hasProx()) {
String proxName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.POS_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.POS_EXTENSION);
posIn = state.directory.openInput(proxName, state.context);
CodecUtil.checkIndexHeader(
posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
@@ -173,7 +169,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene101PostingsFormat.PAY_EXTENSION);
+ Lucene103PostingsFormat.PAY_EXTENSION);
payIn = state.directory.openInput(payName, state.context);
CodecUtil.checkIndexHeader(
payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
@@ -465,7 +461,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
if (needsPos) {
- this.posIn = Lucene101PostingsReader.this.posIn.clone();
+ this.posIn = Lucene103PostingsReader.this.posIn.clone();
posInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(posIn);
posDeltaBuffer = new int[BLOCK_SIZE];
} else {
@@ -475,7 +471,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
if (needsOffsets || needsPayloads) {
- this.payIn = Lucene101PostingsReader.this.payIn.clone();
+ this.payIn = Lucene103PostingsReader.this.payIn.clone();
payInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(payIn);
} else {
this.payIn = null;
@@ -505,7 +501,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
public boolean canReuse(
IndexInput docIn, FieldInfo fieldInfo, int flags, boolean needsImpacts) {
- return docIn == Lucene101PostingsReader.this.docIn
+ return docIn == Lucene103PostingsReader.this.docIn
&& options == fieldInfo.getIndexOptions()
&& indexHasPayloads == fieldInfo.hasPayloads()
&& this.flags == flags
@@ -518,7 +514,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
if (docFreq > 1) {
if (docIn == null) {
// lazy init
- docIn = Lucene101PostingsReader.this.docIn.clone();
+ docIn = Lucene103PostingsReader.this.docIn.clone();
docInUtil = VECTORIZATION_PROVIDER.newPostingDecodingUtil(docIn);
}
prefetchPostings(docIn, termState);
@@ -1063,32 +1059,6 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
}
- @Override
- public int docIDRunEnd() throws IOException {
- // Note: this assumes that BLOCK_SIZE == 128, this bit of the code would need to be changed if
- // the block size was changed.
- // Hack to avoid compiler warning that both sides of the equal sign are identical.
- long blockSize = BLOCK_SIZE;
- assert blockSize == 2 * Long.SIZE;
- boolean level0IsDense =
- encoding == DeltaEncoding.UNARY
- && docBitSet.getBits()[0] == -1L
- && docBitSet.getBits()[1] == -1L;
- if (level0IsDense) {
-
- int level0DocCountUpto = docFreq - docCountLeft;
- boolean level1IsDense =
- level1LastDocID - level0LastDocID == level1DocCountUpto - level0DocCountUpto;
- if (level1IsDense) {
- return level1LastDocID + 1;
- }
-
- return level0LastDocID + 1;
- }
-
- return super.docIDRunEnd();
- }
-
private void skipPositions(int freq) throws IOException {
// Skip positions now:
int toSkip = posPendingCount - freq;
@@ -1332,14 +1302,13 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
@Override
public List<Impact> getImpacts(int level) {
- if (indexHasFreq == false) {
- return DUMMY_IMPACTS_NO_FREQS;
- }
- if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
- return readImpacts(level0SerializedImpacts, level0Impacts);
- }
- if (level == 1) {
- return readImpacts(level1SerializedImpacts, level1Impacts);
+ if (indexHasFreq) {
+ if (level == 0 && level0LastDocID != NO_MORE_DOCS) {
+ return readImpacts(level0SerializedImpacts, level0Impacts);
+ }
+ if (level == 1) {
+ return readImpacts(level1SerializedImpacts, level1Impacts);
+ }
}
return DUMMY_IMPACTS;
}
@@ -1347,7 +1316,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
private List<Impact> readImpacts(BytesRef serialized, MutableImpactList impactsList) {
var scratch = this.scratch;
scratch.reset(serialized.bytes, 0, serialized.length);
- Lucene101PostingsReader.readImpacts(scratch, impactsList);
+ Lucene103PostingsReader.readImpacts(scratch, impactsList);
return impactsList;
}
};
@@ -1360,7 +1329,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
/**
- * @see Lucene101PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)
+ * @see Lucene103PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)
*/
static int readVInt15(DataInput in) throws IOException {
short s = in.readShort();
@@ -1372,7 +1341,7 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
}
/**
- * @see Lucene101PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)
+ * @see Lucene103PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)
*/
static long readVLong15(DataInput in) throws IOException {
short s = in.readShort();
@@ -1389,8 +1358,9 @@ public final class Lucene101PostingsReader extends PostingsReaderBase {
if (docIn.getFilePointer() != state.docStartFP) {
// Don't prefetch if the input is already positioned at the right offset, which suggests that
// the caller is streaming the entire inverted index (e.g. for merging), let the read-ahead
- // logic do its work instead. Note that this heuristic also handles terms with skip data
- // starting in version 912, where skip data was directly inlined into postings lists.
+ // logic do its work instead. Note that this heuristic doesn't work for terms that have skip
+ // data, since skip data is stored after the last term, but handling all terms that have <128
+ // docs is a good start already.
docIn.prefetch(state.docStartFP, 1);
}
// Note: we don't prefetch positions or offsets, which are less likely to be needed.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java
similarity index 95%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java
index 3d19a69b82d..b5d5047e623 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/Lucene101PostingsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/Lucene103PostingsWriter.java
@@ -14,15 +14,15 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.*;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.LEVEL1_MASK;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.META_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.*;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.LEVEL1_MASK;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.META_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.TERMS_CODEC;
import java.io.IOException;
import java.util.Arrays;
@@ -32,7 +32,7 @@ import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Impact;
@@ -49,8 +49,8 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-/** Writer for {@link Lucene101PostingsFormat}. */
-public class Lucene101PostingsWriter extends PushPostingsWriterBase {
+/** Writer for {@link Lucene103PostingsFormat}. */
+public class Lucene103PostingsWriter extends PushPostingsWriterBase {
static final IntBlockTermState EMPTY_STATE = new IntBlockTermState();
@@ -136,19 +136,19 @@ public class Lucene101PostingsWriter extends PushPostingsWriterBase {
private final FixedBitSet spareBitSet = new FixedBitSet(BLOCK_SIZE * Integer.SIZE);
/** Sole public constructor. */
- public Lucene101PostingsWriter(SegmentWriteState state) throws IOException {
- this(state, Lucene101PostingsFormat.VERSION_CURRENT);
+ public Lucene103PostingsWriter(SegmentWriteState state) throws IOException {
+ this(state, Lucene103PostingsFormat.VERSION_CURRENT);
}
/** Constructor that takes a version. */
- Lucene101PostingsWriter(SegmentWriteState state, int version) throws IOException {
+ Lucene103PostingsWriter(SegmentWriteState state, int version) throws IOException {
this.version = version;
String metaFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.META_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.META_EXTENSION);
String docFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.DOC_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.DOC_EXTENSION);
metaOut = state.directory.createOutput(metaFileName, state.context);
IndexOutput posOut = null;
IndexOutput payOut = null;
@@ -165,7 +165,7 @@ public class Lucene101PostingsWriter extends PushPostingsWriterBase {
posDeltaBuffer = new int[BLOCK_SIZE];
String posFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene101PostingsFormat.POS_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene103PostingsFormat.POS_EXTENSION);
posOut = state.directory.createOutput(posFileName, state.context);
CodecUtil.writeIndexHeader(
posOut, POS_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
@@ -191,7 +191,7 @@ public class Lucene101PostingsWriter extends PushPostingsWriterBase {
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene101PostingsFormat.PAY_EXTENSION);
+ Lucene103PostingsFormat.PAY_EXTENSION);
payOut = state.directory.createOutput(payFileName, state.context);
CodecUtil.writeIndexHeader(
payOut, PAY_CODEC, version, state.segmentInfo.getId(), state.segmentSuffix);
@@ -429,12 +429,13 @@ public class Lucene101PostingsWriter extends PushPostingsWriterBase {
// storage-efficient than the next number of bits per value (which effectively slightly biases
// towards the bit set approach).
int bitsPerValue = forDeltaUtil.bitsRequired(docDeltaBuffer);
- int sum = Math.toIntExact(Arrays.stream(docDeltaBuffer).sum());
- int numBitSetLongs = FixedBitSet.bits2words(sum);
+ int docRange = lastDocID - level0LastDocID;
+ assert docRange == Arrays.stream(docDeltaBuffer).sum();
+ int numBitSetLongs = FixedBitSet.bits2words(docRange);
int numBitsNextBitsPerValue = Math.min(Integer.SIZE, bitsPerValue + 1) * BLOCK_SIZE;
- if (sum == BLOCK_SIZE) {
+ if (docRange == BLOCK_SIZE) {
level0Output.writeByte((byte) 0);
- } else if (version < VERSION_DENSE_BLOCKS_AS_BITSETS || numBitsNextBitsPerValue <= sum) {
+ } else if (numBitsNextBitsPerValue <= docRange) {
level0Output.writeByte((byte) bitsPerValue);
forDeltaUtil.encodeDeltas(bitsPerValue, docDeltaBuffer, level0Output);
} else {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/PForUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java
index fd8ecd056b4..81aef85cd03 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PForUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingIndexInput.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingIndexInput.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java
index 59388e3446b..e66ed635146 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingIndexInput.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingsUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingsUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java
index 34431a3689f..e28d59166dc 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/PostingsUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/PostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/CompressionAlgorithm.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/CompressionAlgorithm.java
similarity index 97%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/CompressionAlgorithm.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/CompressionAlgorithm.java
index de24dc3fb05..120d009f012 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/CompressionAlgorithm.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/CompressionAlgorithm.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/FieldReader.java
similarity index 68%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/FieldReader.java
index 259639762ec..ceacb9507f2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/FieldReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/FieldReader.java
@@ -14,23 +14,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
-
-import static org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.OffHeapFSTStore;
/**
* BlockTree's implementation of {@link Terms}.
@@ -46,25 +39,23 @@ public final class FieldReader extends Terms {
final long sumTotalTermFreq;
final long sumDocFreq;
final int docCount;
- final long rootBlockFP;
- final BytesRef rootCode;
final BytesRef minTerm;
final BytesRef maxTerm;
- final Lucene90BlockTreeTermsReader parent;
-
- final FST<BytesRef> index;
+ final long indexStart;
+ final long rootFP;
+ final long indexEnd;
+ final Lucene103BlockTreeTermsReader parent;
+ final IndexInput indexIn;
// private boolean DEBUG;
FieldReader(
- Lucene90BlockTreeTermsReader parent,
+ Lucene103BlockTreeTermsReader parent,
FieldInfo fieldInfo,
long numTerms,
- BytesRef rootCode,
long sumTotalTermFreq,
long sumDocFreq,
int docCount,
- long indexStartFP,
IndexInput metaIn,
IndexInput indexIn,
BytesRef minTerm,
@@ -80,59 +71,20 @@ public final class FieldReader extends Terms {
this.docCount = docCount;
this.minTerm = minTerm;
this.maxTerm = maxTerm;
+
// if (DEBUG) {
// System.out.println("BTTR: seg=" + segment + " field=" + fieldInfo.name + " rootBlockCode="
// + rootCode + " divisor=" + indexDivisor);
// }
- rootBlockFP =
- readVLongOutput(new ByteArrayDataInput(rootCode.bytes, rootCode.offset, rootCode.length))
- >>> Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;
- // Initialize FST always off-heap.
- var metadata = FST.readMetadata(metaIn, ByteSequenceOutputs.getSingleton());
- index = FST.fromFSTReader(metadata, new OffHeapFSTStore(indexIn, indexStartFP, metadata));
- /*
- if (false) {
- final String dotFileName = segment + "_" + fieldInfo.name + ".dot";
- Writer w = new OutputStreamWriter(new FileOutputStream(dotFileName));
- Util.toDot(index, w, false, false);
- System.out.println("FST INDEX: SAVED to " + dotFileName);
- w.close();
- }
- */
- BytesRef emptyOutput = metadata.getEmptyOutput();
- if (rootCode.equals(emptyOutput) == false) {
- // TODO: this branch is never taken
- assert false;
- this.rootCode = rootCode;
- } else {
- this.rootCode = emptyOutput;
- }
- }
- long readVLongOutput(DataInput in) throws IOException {
- if (parent.version >= VERSION_MSB_VLONG_OUTPUT) {
- return readMSBVLong(in);
- } else {
- return in.readVLong();
- }
+ this.indexStart = metaIn.readVLong();
+ this.rootFP = metaIn.readVLong();
+ this.indexEnd = metaIn.readVLong();
+ this.indexIn = indexIn;
}
- /**
- * Decodes a variable length byte[] in MSB order back to long, as written by {@link
- * Lucene90BlockTreeTermsWriter#writeMSBVLong}.
- *
- * <p>Package private for testing.
- */
- static long readMSBVLong(DataInput in) throws IOException {
- long l = 0L;
- while (true) {
- byte b = in.readByte();
- l = (l << 7) | (b & 0x7FL);
- if ((b & 0x80) == 0) {
- break;
- }
- }
- return l;
+ private TrieReader newReader() throws IOException {
+ return new TrieReader(indexIn.slice("trie index", indexStart, indexEnd - indexStart), rootFP);
}
@Override
@@ -158,7 +110,7 @@ public final class FieldReader extends Terms {
/** For debugging -- used by CheckIndex too */
@Override
public Stats getStats() throws IOException {
- return new SegmentTermsEnum(this).computeBlockStats();
+ return new SegmentTermsEnum(this, newReader()).computeBlockStats();
}
@Override
@@ -186,7 +138,7 @@ public final class FieldReader extends Terms {
@Override
public TermsEnum iterator() throws IOException {
- return new SegmentTermsEnum(this);
+ return new SegmentTermsEnum(this, newReader());
}
@Override
@@ -221,6 +173,7 @@ public final class FieldReader extends Terms {
}
return new IntersectTermsEnum(
this,
+ newReader(),
compiled.getTransitionAccessor(),
compiled.getByteRunnable(),
compiled.commonSuffixRef,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/IntersectTermsEnum.java
similarity index 89%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnum.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/IntersectTermsEnum.java
index e2f284f780d..647e2eb1665 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/IntersectTermsEnum.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import org.apache.lucene.index.BaseTermsEnum;
@@ -30,7 +30,6 @@ import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.automaton.ByteRunnable;
import org.apache.lucene.util.automaton.Transition;
import org.apache.lucene.util.automaton.TransitionAccessor;
-import org.apache.lucene.util.fst.FST;
/**
* This is used to implement efficient {@link Terms#intersect} for block-tree. Note that it cannot
@@ -47,8 +46,7 @@ final class IntersectTermsEnum extends BaseTermsEnum {
IntersectTermsEnumFrame[] stack;
- @SuppressWarnings({"rawtypes", "unchecked"})
- private FST.Arc<BytesRef>[] arcs = new FST.Arc[5];
+ private TrieReader.Node[] nodes = new TrieReader.Node[5];
final ByteRunnable runAutomaton;
final TransitionAccessor automaton;
@@ -59,19 +57,17 @@ final class IntersectTermsEnum extends BaseTermsEnum {
private final BytesRef term = new BytesRef();
- private final FST.BytesReader fstReader;
+ final TrieReader trieReader;
final FieldReader fr;
private BytesRef savedStartTerm;
- private final SegmentTermsEnum.OutputAccumulator outputAccumulator =
- new SegmentTermsEnum.OutputAccumulator();
-
// TODO: in some cases we can filter by length? eg
// regexp foo*bar must be at least length 6 bytes
public IntersectTermsEnum(
FieldReader fr,
+ TrieReader trieReader,
TransitionAccessor automaton,
ByteRunnable runAutomaton,
BytesRef commonSuffix,
@@ -91,11 +87,11 @@ final class IntersectTermsEnum extends BaseTermsEnum {
for (int idx = 0; idx < stack.length; idx++) {
stack[idx] = new IntersectTermsEnumFrame(this, idx);
}
- for (int arcIdx = 0; arcIdx < arcs.length; arcIdx++) {
- arcs[arcIdx] = new FST.Arc<>();
+ for (int nodeIdx = 1; nodeIdx < nodes.length; nodeIdx++) {
+ nodes[nodeIdx] = new TrieReader.Node();
}
- fstReader = fr.index.getBytesReader();
+ this.trieReader = trieReader;
// TODO: if the automaton is "smallish" we really
// should use the terms index to seek at least to
@@ -104,23 +100,22 @@ final class IntersectTermsEnum extends BaseTermsEnum {
// Else the seek cost of loading the frames will be
// too costly.
- final FST.Arc<BytesRef> arc = fr.index.getFirstArc(arcs[0]);
+ final TrieReader.Node node = nodes[0] = trieReader.root;
// Empty string prefix must have an output in the index!
- assert arc.isFinal();
+ assert node.hasOutput();
// Special pushFrame since it's the first one:
final IntersectTermsEnumFrame f = stack[0];
- f.fp = f.fpOrig = fr.rootBlockFP;
+ f.fp = f.fpOrig = trieReader.root.outputFp;
f.prefix = 0;
f.setState(0);
- f.arc = arc;
- f.load(fr.rootCode);
+ f.node = node;
+ f.load(node);
// for assert:
assert setSavedStartTerm(startTerm);
currentFrame = f;
- outputAccumulator.push(currentFrame.arc.output());
if (startTerm != null) {
seekToStartTerm(startTerm);
@@ -155,18 +150,17 @@ final class IntersectTermsEnum extends BaseTermsEnum {
return stack[ord];
}
- private FST.Arc<BytesRef> getArc(int ord) {
- if (ord >= arcs.length) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- final FST.Arc<BytesRef>[] next =
- new FST.Arc[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(arcs, 0, next, 0, arcs.length);
- for (int arcOrd = arcs.length; arcOrd < next.length; arcOrd++) {
- next[arcOrd] = new FST.Arc<>();
+ private TrieReader.Node getNode(int ord) {
+ if (ord >= nodes.length) {
+ final TrieReader.Node[] next =
+ new TrieReader.Node[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(nodes, 0, next, 0, nodes.length);
+ for (int nodeOrd = nodes.length; nodeOrd < next.length; nodeOrd++) {
+ next[nodeOrd] = new TrieReader.Node();
}
- arcs = next;
+ nodes = next;
}
- return arcs[ord];
+ return nodes[ord];
}
private IntersectTermsEnumFrame pushFrame(int state) throws IOException {
@@ -178,32 +172,28 @@ final class IntersectTermsEnum extends BaseTermsEnum {
f.prefix = currentFrame.prefix + currentFrame.suffix;
f.setState(state);
- // Walk the arc through the index -- we only
+ // Walk the node through the index -- we only
// "bother" with this so we can get the floor data
// from the index and skip floor blocks when
// possible:
- FST.Arc<BytesRef> arc = currentFrame.arc;
+ TrieReader.Node node = currentFrame.node;
int idx = currentFrame.prefix;
assert currentFrame.suffix > 0;
- int initOutputCount = outputAccumulator.outputCount();
while (idx < f.prefix) {
final int target = term.bytes[idx] & 0xff;
// TODO: we could be more efficient for the next()
- // case by using current arc as starting point,
- // passed to findTargetArc
- arc = fr.index.findTargetArc(target, arc, getArc(1 + idx), fstReader);
- assert arc != null;
- outputAccumulator.push(arc.output());
+ // case by using current node as starting point,
+ // passed to findTargetNode
+ TrieReader.Node parent = node;
+ node = trieReader.lookupChild(target, parent, getNode(1 + idx));
+ assert node != null;
idx++;
}
- f.arc = arc;
- f.outputNum = outputAccumulator.outputCount() - initOutputCount;
- assert arc.isFinal();
- outputAccumulator.push(arc.nextFinalOutput());
- f.load(outputAccumulator);
- outputAccumulator.pop(arc.nextFinalOutput());
+ f.node = node;
+ assert node.hasOutput();
+ f.load(node);
return f;
}
@@ -256,8 +246,8 @@ final class IntersectTermsEnum extends BaseTermsEnum {
if (term.length < target.length) {
term.bytes = ArrayUtil.grow(term.bytes, target.length);
}
- FST.Arc<BytesRef> arc = arcs[0];
- assert arc == currentFrame.arc;
+ TrieReader.Node node = nodes[0];
+ assert node == currentFrame.node;
for (int idx = 0; idx <= target.length; idx++) {
@@ -346,7 +336,6 @@ final class IntersectTermsEnum extends BaseTermsEnum {
throw NoMoreTermsException.INSTANCE;
}
final long lastFP = currentFrame.fpOrig;
- outputAccumulator.pop(currentFrame.outputNum);
currentFrame = stack[currentFrame.ord - 1];
currentTransition = currentFrame.transition;
assert currentFrame.lastSubFP == lastFP;
@@ -433,7 +422,6 @@ final class IntersectTermsEnum extends BaseTermsEnum {
currentFrame = null;
return null;
}
- outputAccumulator.pop(currentFrame.outputNum);
currentFrame = stack[currentFrame.ord - 1];
currentTransition = currentFrame.transition;
isSubBlock = popPushNext();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/IntersectTermsEnumFrame.java
similarity index 91%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/IntersectTermsEnumFrame.java
index 9f6bb75788e..8bb6f84b798 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/IntersectTermsEnumFrame.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/IntersectTermsEnumFrame.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import java.util.Arrays;
@@ -22,10 +22,9 @@ import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.automaton.Transition;
-import org.apache.lucene.util.fst.FST;
// TODO: can we share this with the frame in STE?
final class IntersectTermsEnumFrame {
@@ -55,7 +54,8 @@ final class IntersectTermsEnumFrame {
int statsSingletonRunLength = 0;
final ByteArrayDataInput statsReader = new ByteArrayDataInput();
- final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();
+ long floorDataPos;
+ IndexInput floorDataReader;
// Length of prefix shared by all terms in this block
int prefix;
@@ -80,7 +80,7 @@ final class IntersectTermsEnumFrame {
int transitionIndex;
int transitionCount;
- FST.Arc<BytesRef> arc;
+ TrieReader.Node node;
final BlockTermState termState;
@@ -89,8 +89,6 @@ final class IntersectTermsEnumFrame {
final ByteArrayDataInput bytesReader = new ByteArrayDataInput();
- int outputNum;
-
int startBytePos;
int suffix;
@@ -108,6 +106,7 @@ final class IntersectTermsEnumFrame {
void loadNextFloorBlock() throws IOException {
assert numFollowFloorBlocks > 0 : "nextFloorLabel=" + nextFloorLabel;
+ floorDataReader.seek(floorDataPos);
do {
fp = fpOrig + (floorDataReader.readVLong() >>> 1);
numFollowFloorBlocks--;
@@ -117,8 +116,8 @@ final class IntersectTermsEnumFrame {
nextFloorLabel = 256;
}
} while (numFollowFloorBlocks != 0 && nextFloorLabel <= transition.min);
-
- load((Long) null);
+ load(null);
+ floorDataPos = floorDataReader.getFilePointer();
}
public void setState(int state) {
@@ -139,23 +138,12 @@ final class IntersectTermsEnumFrame {
}
}
- void load(BytesRef frameIndexData) throws IOException {
- floorDataReader.reset(frameIndexData.bytes, frameIndexData.offset, frameIndexData.length);
- load(ite.fr.readVLongOutput(floorDataReader));
- }
-
- void load(SegmentTermsEnum.OutputAccumulator outputAccumulator) throws IOException {
- outputAccumulator.prepareRead();
- long code = ite.fr.readVLongOutput(outputAccumulator);
- outputAccumulator.setFloorData(floorDataReader);
- load(code);
- }
-
- void load(Long blockCode) throws IOException {
- if (blockCode != null) {
+ void load(TrieReader.Node node) throws IOException {
+ if (node != null) {
// This block is the first one in a possible sequence of floor blocks corresponding to a
- // single seek point from the FST terms index
- if ((blockCode & Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0) {
+ // single seek point from the trie terms index
+ if (node.isFloor()) {
+ floorDataReader = node.floorData(ite.trieReader);
// Floor frame
numFollowFloorBlocks = floorDataReader.readVInt();
nextFloorLabel = floorDataReader.readByte() & 0xff;
@@ -175,6 +163,7 @@ final class IntersectTermsEnumFrame {
}
}
}
+ floorDataPos = floorDataReader.getFilePointer();
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsReader.java
similarity index 90%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsReader.java
index 3e2ddbbb5d5..b4c5e298098 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsReader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import java.util.ArrayList;
@@ -38,8 +38,6 @@ import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.ReadAdvice;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.Outputs;
/**
* A block-based terms index and dictionary that assigns terms to variable length blocks according
@@ -59,20 +57,11 @@ import org.apache.lucene.util.fst.Outputs;
* <p>Use {@link org.apache.lucene.index.CheckIndex} with the <code>-verbose</code> option to see
* summary statistics on the blocks in the dictionary.
*
- * <p>See {@link Lucene90BlockTreeTermsWriter}.
+ * <p>See {@link Lucene103BlockTreeTermsWriter}.
*
* @lucene.experimental
*/
-public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
-
- static final Outputs<BytesRef> FST_OUTPUTS = ByteSequenceOutputs.getSingleton();
-
- static final BytesRef NO_OUTPUT = FST_OUTPUTS.getNoOutput();
-
- static final int OUTPUT_FLAGS_NUM_BITS = 2;
- static final int OUTPUT_FLAGS_MASK = 0x3;
- static final int OUTPUT_FLAG_IS_FLOOR = 0x1;
- static final int OUTPUT_FLAG_HAS_TERMS = 0x2;
+public final class Lucene103BlockTreeTermsReader extends FieldsProducer {
/** Extension of terms file */
static final String TERMS_EXTENSION = "tim";
@@ -82,16 +71,8 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
/** Initial terms format. */
public static final int VERSION_START = 0;
- /**
- * Version that encode output as MSB VLong for better outputs sharing in FST, see GITHUB#12620.
- */
- public static final int VERSION_MSB_VLONG_OUTPUT = 1;
-
- /** The version that specialize arc store for continuous label in FST. */
- public static final int VERSION_FST_CONTINUOUS_ARCS = 2;
-
/** Current terms format. */
- public static final int VERSION_CURRENT = VERSION_FST_CONTINUOUS_ARCS;
+ public static final int VERSION_CURRENT = VERSION_START;
/** Extension of terms index file */
static final String TERMS_INDEX_EXTENSION = "tip";
@@ -123,7 +104,7 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
final int version;
/** Sole constructor. */
- public Lucene90BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state)
+ public Lucene103BlockTreeTermsReader(PostingsReaderBase postingsReader, SegmentReadState state)
throws IOException {
boolean success = false;
@@ -184,7 +165,6 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
throw new CorruptIndexException(
"Illegal numTerms for field number: " + field, metaIn);
}
- final BytesRef rootCode = readBytesRef(metaIn);
final FieldInfo fieldInfo = state.fieldInfos.fieldInfo(field);
if (fieldInfo == null) {
throw new CorruptIndexException("invalid field number: " + field, metaIn);
@@ -219,7 +199,6 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
"invalid sumTotalTermFreq: " + sumTotalTermFreq + " sumDocFreq: " + sumDocFreq,
metaIn);
}
- final long indexStartFP = metaIn.readVLong();
FieldReader previous =
fieldMap.put(
fieldInfo.number,
@@ -227,11 +206,9 @@ public final class Lucene90BlockTreeTermsReader extends FieldsProducer {
this,
fieldInfo,
numTerms,
- rootCode,
sumTotalTermFreq,
sumDocFreq,
docCount,
- indexStartFP,
metaIn,
indexIn,
minTerm,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java
similarity index 85%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java
index 8d0d5aaaa22..ee4689183ab 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Lucene103BlockTreeTermsWriter.java
@@ -14,9 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
-
-import static org.apache.lucene.util.fst.FSTCompiler.getOnHeapReaderWriter;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import java.util.ArrayList;
@@ -45,17 +43,10 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.IntsRefBuilder;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.compress.LZ4;
import org.apache.lucene.util.compress.LowercaseAsciiCompression;
-import org.apache.lucene.util.fst.ByteSequenceOutputs;
-import org.apache.lucene.util.fst.BytesRefFSTEnum;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.FSTCompiler;
-import org.apache.lucene.util.fst.Util;
-import org.apache.lucene.util.packed.PackedInts;
/*
TODO:
@@ -153,21 +144,21 @@ import org.apache.lucene.util.packed.PackedInts;
*
* <h2>Term Metadata</h2>
*
- * <p>The .tmd file contains the list of term metadata (such as FST index metadata) and field level
+ * <p>The .tmd file contains the list of term metadata (such as trie index metadata) and field level
* statistics (such as sum of total term freq).
*
* <ul>
* <li>TermsMeta (.tmd) --> Header, NumFields, <FieldStats><sup>NumFields</sup>,
* TermIndexLength, TermDictLength, Footer
* <li>FieldStats --> FieldNumber, NumTerms, RootCodeLength, Byte<sup>RootCodeLength</sup>,
- * SumTotalTermFreq?, SumDocFreq, DocCount, MinTerm, MaxTerm, IndexStartFP, FSTHeader,
- * <i>FSTMetadata</i>
- * <li>Header,FSTHeader --> {@link CodecUtil#writeHeader CodecHeader}
+ * SumTotalTermFreq?, SumDocFreq, DocCount, MinTerm, MaxTerm, IndexStartFP, TrieRootNodeFp,
+ * IndexEndFp
+ * <li>Header --> {@link CodecUtil#writeHeader CodecHeader}
* <li>TermIndexLength, TermDictLength --> {@link DataOutput#writeLong Uint64}
* <li>MinTerm,MaxTerm --> {@link DataOutput#writeVInt VInt} length followed by the byte[]
* <li>NumFields,FieldNumber,RootCodeLength,DocCount --> {@link DataOutput#writeVInt VInt}
- * <li>NumTerms,SumTotalTermFreq,SumDocFreq,IndexStartFP --> {@link DataOutput#writeVLong
- * VLong}
+ * <li>NumTerms,SumTotalTermFreq,SumDocFreq,IndexStartFP,TrieRootNodeFp,IndexEndFp --> {@link
+ * DataOutput#writeVLong VLong}
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}
* </ul>
*
@@ -192,39 +183,42 @@ import org.apache.lucene.util.packed.PackedInts;
* saving a disk seek.
*
* <ul>
- * <li>TermsIndex (.tip) --> Header, FSTIndex<sup>NumFields</sup>Footer
+ * <li>TermsIndex (.tip) --> Header, TrieIndex<sup>NumFields</sup>Footer
* <li>Header --> {@link CodecUtil#writeHeader CodecHeader}
- * <!-- TODO: better describe FST output here -->
- * <li>FSTIndex --> {@link FST FST<byte[]>}
+ * <!-- TODO: better describe trie output here -->
+ * <li>TrieIndex --> {trie<byte[]>}
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}
* </ul>
*
* <p>Notes:
*
* <ul>
- * <li>The .tip file contains a separate FST for each field. The FST maps a term prefix to the
+ * <li>The .tip file contains a separate trie for each field. The trie maps a term prefix to the
* on-disk block that holds all terms starting with that prefix. Each field's IndexStartFP
- * points to its FST.
+ * points to its trie.
+ * <li>The trie stores its nodes in a depth-first order. Each node stores its output (if any) and
+ * children's labels and file pointers. Nodes have various strategies to store their children,
+ * decided by the distribution of the children's labels.
* <li>It's possible that an on-disk block would contain too many terms (more than the allowed
* maximum (default: 48)). When this happens, the block is sub-divided into new blocks (called
- * "floor blocks"), and then the output in the FST for the block's prefix encodes the leading
+ * "floor blocks"), and then the output in the trie for the block's prefix encodes the leading
* byte of each sub-block, and its file pointer.
* </ul>
*
- * @see Lucene90BlockTreeTermsReader
+ * @see Lucene103BlockTreeTermsReader
* @lucene.experimental
*/
-public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
+public final class Lucene103BlockTreeTermsWriter extends FieldsConsumer {
/**
* Suggested default value for the {@code minItemsInBlock} parameter to {@link
- * #Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.
+ * #Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.
*/
public static final int DEFAULT_MIN_BLOCK_SIZE = 25;
/**
* Suggested default value for the {@code maxItemsInBlock} parameter to {@link
- * #Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.
+ * #Lucene103BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)}.
*/
public static final int DEFAULT_MAX_BLOCK_SIZE = 48;
@@ -251,7 +245,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
* minItemsPerBlock and maxItemsPerBlock, though in some cases the blocks may be smaller than the
* min.
*/
- public Lucene90BlockTreeTermsWriter(
+ public Lucene103BlockTreeTermsWriter(
SegmentWriteState state,
PostingsWriterBase postingsWriter,
int minItemsInBlock,
@@ -262,11 +256,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
postingsWriter,
minItemsInBlock,
maxItemsInBlock,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT);
+ Lucene103BlockTreeTermsReader.VERSION_CURRENT);
}
/** Expert constructor that allows configuring the version, used for bw tests. */
- public Lucene90BlockTreeTermsWriter(
+ public Lucene103BlockTreeTermsWriter(
SegmentWriteState state,
PostingsWriterBase postingsWriter,
int minItemsInBlock,
@@ -277,13 +271,13 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
- if (version < Lucene90BlockTreeTermsReader.VERSION_START
- || version > Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+ if (version < Lucene103BlockTreeTermsReader.VERSION_START
+ || version > Lucene103BlockTreeTermsReader.VERSION_CURRENT) {
throw new IllegalArgumentException(
"Expected version in range ["
- + Lucene90BlockTreeTermsReader.VERSION_START
+ + Lucene103BlockTreeTermsReader.VERSION_START
+ ", "
- + Lucene90BlockTreeTermsReader.VERSION_CURRENT
+ + Lucene103BlockTreeTermsReader.VERSION_CURRENT
+ "], but got "
+ version);
}
@@ -297,14 +291,14 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene90BlockTreeTermsReader.TERMS_EXTENSION);
+ Lucene103BlockTreeTermsReader.TERMS_EXTENSION);
termsOut = state.directory.createOutput(termsName, state.context);
boolean success = false;
IndexOutput metaOut = null, indexOut = null;
try {
CodecUtil.writeIndexHeader(
termsOut,
- Lucene90BlockTreeTermsReader.TERMS_CODEC_NAME,
+ Lucene103BlockTreeTermsReader.TERMS_CODEC_NAME,
version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -313,11 +307,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene90BlockTreeTermsReader.TERMS_INDEX_EXTENSION);
+ Lucene103BlockTreeTermsReader.TERMS_INDEX_EXTENSION);
indexOut = state.directory.createOutput(indexName, state.context);
CodecUtil.writeIndexHeader(
indexOut,
- Lucene90BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
+ Lucene103BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -327,11 +321,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene90BlockTreeTermsReader.TERMS_META_EXTENSION);
+ Lucene103BlockTreeTermsReader.TERMS_META_EXTENSION);
metaOut = state.directory.createOutput(metaName, state.context);
CodecUtil.writeIndexHeader(
metaOut,
- Lucene90BlockTreeTermsReader.TERMS_META_CODEC_NAME,
+ Lucene103BlockTreeTermsReader.TERMS_META_CODEC_NAME,
version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -405,13 +399,6 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
}
}
- static long encodeOutput(long fp, boolean hasTerms, boolean isFloor) {
- assert fp < (1L << 62);
- return (fp << 2)
- | (hasTerms ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- | (isFloor ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);
- }
-
private static class PendingEntry {
public final boolean isTerm;
@@ -438,30 +425,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
}
}
- /**
- * Encodes long value to variable length byte[], in MSB order. Use {@link
- * FieldReader#readMSBVLong} to decode.
- *
- * <p>Package private for testing
- */
- static void writeMSBVLong(long l, DataOutput scratchBytes) throws IOException {
- assert l >= 0;
- // Keep zero bits on most significant byte to have more chance to get prefix bytes shared.
- // e.g. we expect 0x7FFF stored as [0x81, 0xFF, 0x7F] but not [0xFF, 0xFF, 0x40]
- final int bytesNeeded = (Long.SIZE - Long.numberOfLeadingZeros(l) - 1) / 7 + 1;
- l <<= Long.SIZE - bytesNeeded * 7;
- for (int i = 1; i < bytesNeeded; i++) {
- scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL) | 0x80));
- l = l << 7;
- }
- scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL)));
- }
-
private final class PendingBlock extends PendingEntry {
public final BytesRef prefix;
public final long fp;
- public FST<BytesRef> index;
- public List<FST<BytesRef>> subIndices;
+ public TrieBuilder index;
+ public List<TrieBuilder> subIndices;
public final boolean hasTerms;
public final boolean isFloor;
public final int floorLeadByte;
@@ -472,7 +440,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
boolean hasTerms,
boolean isFloor,
int floorLeadByte,
- List<FST<BytesRef>> subIndices) {
+ List<TrieBuilder> subIndices) {
super(false);
this.prefix = prefix;
this.fp = fp;
@@ -487,10 +455,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
return "BLOCK: prefix=" + ToStringUtils.bytesRefToString(prefix);
}
- public void compileIndex(
- List<PendingBlock> blocks,
- ByteBuffersDataOutput scratchBytes,
- IntsRefBuilder scratchIntsRef)
+ public void compileIndex(List<PendingBlock> blocks, ByteBuffersDataOutput scratchBytes)
throws IOException {
assert (isFloor && blocks.size() > 1) || (isFloor == false && blocks.size() == 1)
@@ -499,12 +464,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
assert scratchBytes.size() == 0;
- // write the leading vLong in MSB order for better outputs sharing in the FST
- if (version >= Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT) {
- writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
- } else {
- scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
- }
+ BytesRef floorData = null;
if (isFloor) {
scratchBytes.writeVInt(blocks.size() - 1);
for (int i = 1; i < blocks.size(); i++) {
@@ -518,54 +478,24 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
assert sub.fp > fp;
scratchBytes.writeVLong((sub.fp - fp) << 1 | (sub.hasTerms ? 1 : 0));
}
+ floorData = new BytesRef(scratchBytes.toArrayCopy());
}
- long estimateSize = prefix.length;
- for (PendingBlock block : blocks) {
- if (block.subIndices != null) {
- for (FST<BytesRef> subIndex : block.subIndices) {
- estimateSize += subIndex.numBytes();
- }
- }
- }
- int estimateBitsRequired = PackedInts.bitsRequired(estimateSize);
- int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));
-
- final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
- final int fstVersion;
- if (version >= Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
- fstVersion = FST.VERSION_CURRENT;
- } else {
- fstVersion = FST.VERSION_90;
- }
- final FSTCompiler<BytesRef> fstCompiler =
- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
- // Disable suffixes sharing for block tree index because suffixes are mostly dropped
- // from the FST index and left in the term blocks.
- .suffixRAMLimitMB(0d)
- .dataOutput(getOnHeapReaderWriter(pageBits))
- .setVersion(fstVersion)
- .build();
- // if (DEBUG) {
- // System.out.println(" compile index for prefix=" + prefix);
- // }
- // indexBuilder.DEBUG = false;
- final byte[] bytes = scratchBytes.toArrayCopy();
- assert bytes.length > 0;
- fstCompiler.add(Util.toIntsRef(prefix, scratchIntsRef), new BytesRef(bytes, 0, bytes.length));
+ TrieBuilder trieBuilder =
+ TrieBuilder.bytesRefToTrie(prefix, new TrieBuilder.Output(fp, hasTerms, floorData));
scratchBytes.reset();
// Copy over index for all sub-blocks
for (PendingBlock block : blocks) {
if (block.subIndices != null) {
- for (FST<BytesRef> subIndex : block.subIndices) {
- append(fstCompiler, subIndex, scratchIntsRef);
+ for (TrieBuilder subIndex : block.subIndices) {
+ trieBuilder.append(subIndex);
}
block.subIndices = null;
}
}
- index = FST.fromFSTReader(fstCompiler.compile(), fstCompiler.getFSTReader());
+ index = trieBuilder;
assert subIndices == null;
@@ -576,27 +506,9 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
w.close();
*/
}
-
- // TODO: maybe we could add bulk-add method to
- // Builder? Takes FST and unions it w/ current
- // FST.
- private void append(
- FSTCompiler<BytesRef> fstCompiler, FST<BytesRef> subIndex, IntsRefBuilder scratchIntsRef)
- throws IOException {
- final BytesRefFSTEnum<BytesRef> subIndexEnum = new BytesRefFSTEnum<>(subIndex);
- BytesRefFSTEnum.InputOutput<BytesRef> indexEnt;
- while ((indexEnt = subIndexEnum.next()) != null) {
- // if (DEBUG) {
- // System.out.println(" add sub=" + indexEnt.input + " " + indexEnt.input + " output="
- // + indexEnt.output);
- // }
- fstCompiler.add(Util.toIntsRef(indexEnt.input, scratchIntsRef), indexEnt.output);
- }
- }
}
private final ByteBuffersDataOutput scratchBytes = ByteBuffersDataOutput.newResettableInstance();
- private final IntsRefBuilder scratchIntsRef = new IntsRefBuilder();
private static class StatsWriter {
@@ -771,7 +683,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
assert firstBlock.isFloor || newBlocks.size() == 1;
- firstBlock.compileIndex(newBlocks, scratchBytes, scratchIntsRef);
+ firstBlock.compileIndex(newBlocks, scratchBytes);
// Remove slice from the top of the pending stack, that we just wrote:
pending.subList(pending.size() - count, pending.size()).clear();
@@ -850,7 +762,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
// System.out.println(" isLeaf=" + isLeafBlock);
- final List<FST<BytesRef>> subIndices;
+ final List<TrieBuilder> subIndices;
boolean absolute = true;
@@ -1165,16 +1077,13 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
: "pending.size()=" + pending.size() + " pending=" + pending;
final PendingBlock root = (PendingBlock) pending.get(0);
assert root.prefix.length == 0;
- final BytesRef rootCode = root.index.getEmptyOutput();
- assert rootCode != null;
+ assert root.index.getEmptyOutput() != null;
ByteBuffersDataOutput metaOut = new ByteBuffersDataOutput();
fields.add(metaOut);
metaOut.writeVInt(fieldInfo.number);
metaOut.writeVLong(numTerms);
- metaOut.writeVInt(rootCode.length);
- metaOut.writeBytes(rootCode.bytes, rootCode.offset, rootCode.length);
assert fieldInfo.getIndexOptions() != IndexOptions.NONE;
if (fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
metaOut.writeVLong(sumTotalTermFreq);
@@ -1183,10 +1092,8 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
metaOut.writeVInt(docsSeen.cardinality());
writeBytesRef(metaOut, new BytesRef(firstPendingTerm.termBytes));
writeBytesRef(metaOut, new BytesRef(lastPendingTerm.termBytes));
- metaOut.writeVLong(indexOut.getFilePointer());
- // Write FST to index
root.index.save(metaOut, indexOut);
- // System.out.println(" write FST " + indexStartFP + " field=" + fieldInfo.name);
+ // System.out.println(" write trie " + indexStartFP + " field=" + fieldInfo.name);
/*
if (DEBUG) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/SegmentTermsEnum.java
similarity index 74%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/SegmentTermsEnum.java
index 45ec4ee06ba..99acaae560d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnum.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/SegmentTermsEnum.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import java.io.PrintStream;
@@ -24,35 +24,27 @@ import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.PostingsEnum;
import org.apache.lucene.index.TermState;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
import org.apache.lucene.util.IOBooleanSupplier;
import org.apache.lucene.util.RamUsageEstimator;
-import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.Util;
/** Iterates through terms in this field. */
final class SegmentTermsEnum extends BaseTermsEnum {
// Lazy init:
IndexInput in;
-
- private SegmentTermsEnumFrame[] stack;
+ private SegmentTermsEnumFrame[] stack = new SegmentTermsEnumFrame[0];
private final SegmentTermsEnumFrame staticFrame;
SegmentTermsEnumFrame currentFrame;
boolean termExists;
final FieldReader fr;
-
private int targetBeforeCurrentLength;
// static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
- private final OutputAccumulator outputAccumulator = new OutputAccumulator();
-
// What prefix of the current term was present in the index; when we only next() through the
// index, this stays at 0. It's only set when
// we seekCeil/Exact:
@@ -62,43 +54,17 @@ final class SegmentTermsEnum extends BaseTermsEnum {
private boolean eof;
final BytesRefBuilder term = new BytesRefBuilder();
- private final FST.BytesReader fstReader;
+ private final TrieReader trieReader;
+ private TrieReader.Node[] nodes = new TrieReader.Node[1];
- @SuppressWarnings({"rawtypes", "unchecked"})
- private FST.Arc<BytesRef>[] arcs = new FST.Arc[1];
-
- public SegmentTermsEnum(FieldReader fr) throws IOException {
+ public SegmentTermsEnum(FieldReader fr, TrieReader reader) throws IOException {
this.fr = fr;
-
- // if (DEBUG) {
- // System.out.println("BTTR.init seg=" + fr.parent.segment);
- // }
- stack = new SegmentTermsEnumFrame[0];
-
// Used to hold seek by TermState, or cached seek
staticFrame = new SegmentTermsEnumFrame(this, -1);
-
- if (fr.index == null) {
- fstReader = null;
- } else {
- fstReader = fr.index.getBytesReader();
- }
-
- // Init w/ root block; don't use index since it may
- // not (and need not) have been loaded
- for (int arcIdx = 0; arcIdx < arcs.length; arcIdx++) {
- arcs[arcIdx] = new FST.Arc<>();
- }
-
+ trieReader = reader;
currentFrame = staticFrame;
- final FST.Arc<BytesRef> arc;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
+ nodes[0] = trieReader.root;
+
// currentFrame = pushFrame(arc, rootCode, 0);
// currentFrame.loadBlock();
validIndexPrefix = 0;
@@ -120,25 +86,14 @@ final class SegmentTermsEnum extends BaseTermsEnum {
/** Runs next() through the entire terms dict, computing aggregate statistics. */
public Stats computeBlockStats() throws IOException {
-
Stats stats = new Stats(fr.parent.segment, fr.fieldInfo.name);
- if (fr.index != null) {
- stats.indexNumBytes = fr.index.ramBytesUsed();
- }
currentFrame = staticFrame;
- FST.Arc<BytesRef> arc;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
+ TrieReader.Node node = nodes[0] = trieReader.root;
// Empty string prefix must have an output in the
// index!
- currentFrame = pushFrame(arc, fr.rootCode, 0);
+ currentFrame = pushFrame(node, 0);
currentFrame.fpOrig = currentFrame.fp;
currentFrame.loadBlock();
validIndexPrefix = 0;
@@ -190,14 +145,12 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// Put root frame back:
currentFrame = staticFrame;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
- currentFrame = pushFrame(arc, fr.rootCode, 0);
+
+ node = nodes[0] = trieReader.root;
+ // Empty string prefix must have an output in the index!
+ assert node.hasOutput();
+
+ currentFrame = pushFrame(node, 0);
currentFrame.rewind();
currentFrame.loadBlock();
validIndexPrefix = 0;
@@ -221,49 +174,38 @@ final class SegmentTermsEnum extends BaseTermsEnum {
return stack[ord];
}
- private FST.Arc<BytesRef> getArc(int ord) {
- if (ord >= arcs.length) {
- @SuppressWarnings({"rawtypes", "unchecked"})
- final FST.Arc<BytesRef>[] next =
- new FST.Arc[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
- System.arraycopy(arcs, 0, next, 0, arcs.length);
- for (int arcOrd = arcs.length; arcOrd < next.length; arcOrd++) {
- next[arcOrd] = new FST.Arc<>();
+ private TrieReader.Node getNode(int ord) {
+ if (ord >= nodes.length) {
+ final TrieReader.Node[] next =
+ new TrieReader.Node[ArrayUtil.oversize(1 + ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ System.arraycopy(nodes, 0, next, 0, nodes.length);
+ for (int nodeOrd = nodes.length; nodeOrd < next.length; nodeOrd++) {
+ next[nodeOrd] = new TrieReader.Node();
}
- arcs = next;
+ nodes = next;
}
- return arcs[ord];
- }
-
- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, BytesRef frameData, int length)
- throws IOException {
- outputAccumulator.reset();
- outputAccumulator.push(frameData);
- return pushFrame(arc, length);
+ return nodes[ord];
}
// Pushes a frame we seek'd to
- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, int length) throws IOException {
- outputAccumulator.prepareRead();
- final long code = fr.readVLongOutput(outputAccumulator);
- final long fpSeek = code >>> Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS;
+ SegmentTermsEnumFrame pushFrame(TrieReader.Node node, int length) throws IOException {
final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);
- f.hasTerms = (code & Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS) != 0;
+ f.hasTerms = node.hasTerms;
f.hasTermsOrig = f.hasTerms;
- f.isFloor = (code & Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR) != 0;
+ f.isFloor = node.isFloor();
if (f.isFloor) {
- f.setFloorData(outputAccumulator);
+ f.setFloorData(node.floorData(trieReader));
}
- pushFrame(arc, fpSeek, length);
+ pushFrame(node, node.outputFp, length);
return f;
}
// Pushes next'd frame or seek'd frame; we later
// lazy-load the frame only when needed
- SegmentTermsEnumFrame pushFrame(FST.Arc<BytesRef> arc, long fp, int length) throws IOException {
+ SegmentTermsEnumFrame pushFrame(TrieReader.Node node, long fp, int length) throws IOException {
final SegmentTermsEnumFrame f = getFrame(1 + currentFrame.ord);
- f.arc = arc;
+ f.node = node;
if (f.fpOrig == fp && f.nextEnt != -1) {
// if (DEBUG) System.out.println(" push reused frame ord=" + f.ord + " fp=" + f.fp +
// " isFloor?=" + f.isFloor + " hasTerms=" + f.hasTerms + " pref=" + term + " nextEnt=" +
@@ -309,10 +251,6 @@ final class SegmentTermsEnum extends BaseTermsEnum {
}
private IOBooleanSupplier prepareSeekExact(BytesRef target, boolean prefetch) throws IOException {
- if (fr.index == null) {
- throw new IllegalStateException("terms index was not loaded");
- }
-
if (fr.size() > 0 && (target.compareTo(fr.getMin()) < 0 || target.compareTo(fr.getMax()) > 0)) {
return null;
}
@@ -329,11 +267,10 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// printSeekState(System.out);
// }
- FST.Arc<BytesRef> arc;
+ TrieReader.Node node;
int targetUpto;
targetBeforeCurrentLength = currentFrame.ord;
- outputAccumulator.reset();
if (currentFrame != staticFrame) {
@@ -348,9 +285,8 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);
// }
- arc = arcs[0];
- assert arc.isFinal();
- outputAccumulator.push(arc.output());
+ node = nodes[0];
+ assert node.hasOutput();
targetUpto = 0;
SegmentTermsEnumFrame lastFrame = stack[0];
@@ -366,21 +302,21 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// if (DEBUG) {
// System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit
// + ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) +
- // " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")" + " arc.output=" + arc.output
+ // " vs termLabel=" + (char) (term.bytes[targetUpto]) + ")" + " node.output=" +
+ // node.output
// + " output=" + output);
// }
if (cmp != 0) {
break;
}
- arc = arcs[1 + targetUpto];
- assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF)
- : "arc.label="
- + (char) arc.label()
+ node = nodes[1 + targetUpto];
+ assert node.label == (target.bytes[target.offset + targetUpto] & 0xFF)
+ : "node.label="
+ + (char) node.label
+ " targetLabel="
+ (char) (target.bytes[target.offset + targetUpto] & 0xFF);
- outputAccumulator.push(arc.output());
- if (arc.isFinal()) {
+ if (node.hasOutput()) {
lastFrame = stack[1 + lastFrame.ord];
}
targetUpto++;
@@ -388,7 +324,7 @@ final class SegmentTermsEnum extends BaseTermsEnum {
if (cmp == 0) {
// Second compare the rest of the term, but
- // don't save arc/output/frame; we only do this
+ // don't save node/output/frame; we only do this
// to find out if the target term is before,
// equal or after the current term
cmp =
@@ -444,25 +380,20 @@ final class SegmentTermsEnum extends BaseTermsEnum {
} else {
targetBeforeCurrentLength = -1;
- arc = fr.index.getFirstArc(arcs[0]);
+ node = trieReader.root;
// Empty string prefix must have an output (block) in the index!
- assert arc.isFinal();
- assert arc.output() != null;
+ assert node.hasOutput();
// if (DEBUG) {
// System.out.println(" no seek state; push root frame");
// }
- outputAccumulator.push(arc.output());
-
currentFrame = staticFrame;
// term.length = 0;
targetUpto = 0;
- outputAccumulator.push(arc.nextFinalOutput());
- currentFrame = pushFrame(arc, 0);
- outputAccumulator.pop(arc.nextFinalOutput());
+ currentFrame = pushFrame(node, 0);
}
// if (DEBUG) {
@@ -477,10 +408,10 @@ final class SegmentTermsEnum extends BaseTermsEnum {
final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;
- final FST.Arc<BytesRef> nextArc =
- fr.index.findTargetArc(targetLabel, arc, getArc(1 + targetUpto), fstReader);
+ final TrieReader.Node nextNode =
+ trieReader.lookupChild(targetLabel, node, getNode(1 + targetUpto));
- if (nextArc == null) {
+ if (nextNode == null) {
// Index is exhausted
// if (DEBUG) {
@@ -525,24 +456,20 @@ final class SegmentTermsEnum extends BaseTermsEnum {
}
};
} else {
- // Follow this arc
- arc = nextArc;
+ // Follow this node
+ node = nextNode;
term.setByteAt(targetUpto, (byte) targetLabel);
// Aggregate output as we go:
- assert arc.output() != null;
- outputAccumulator.push(arc.output());
// if (DEBUG) {
// System.out.println(" index: follow label=" + toHex(target.bytes[target.offset +
- // targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);
+ // targetUpto]&0xff) + " node.output=" + node.output + " node.nfo=" + node.nextFinalOutput);
// }
targetUpto++;
- if (arc.isFinal()) {
- // if (DEBUG) System.out.println(" arc is final!");
- outputAccumulator.push(arc.nextFinalOutput());
- currentFrame = pushFrame(arc, targetUpto);
- outputAccumulator.pop(arc.nextFinalOutput());
+ if (node.hasOutput()) {
+ // if (DEBUG) System.out.println(" node is final!");
+ currentFrame = pushFrame(node, targetUpto);
// if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" +
// currentFrame.hasTerms);
}
@@ -602,10 +529,6 @@ final class SegmentTermsEnum extends BaseTermsEnum {
@Override
public SeekStatus seekCeil(BytesRef target) throws IOException {
- if (fr.index == null) {
- throw new IllegalStateException("terms index was not loaded");
- }
-
term.grow(1 + target.length);
assert clearEOF();
@@ -618,11 +541,10 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// printSeekState(System.out);
// }
- FST.Arc<BytesRef> arc;
+ TrieReader.Node node;
int targetUpto;
targetBeforeCurrentLength = currentFrame.ord;
- outputAccumulator.reset();
if (currentFrame != staticFrame) {
@@ -637,9 +559,8 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// System.out.println(" re-use current seek state validIndexPrefix=" + validIndexPrefix);
// }
- arc = arcs[0];
- assert arc.isFinal();
- outputAccumulator.push(arc.output());
+ node = nodes[0];
+ assert node.hasOutput();
targetUpto = 0;
SegmentTermsEnumFrame lastFrame = stack[0];
@@ -655,21 +576,21 @@ final class SegmentTermsEnum extends BaseTermsEnum {
// if (DEBUG) {
// System.out.println(" cycle targetUpto=" + targetUpto + " (vs limit=" + targetLimit +
// ") cmp=" + cmp + " (targetLabel=" + (char) (target.bytes[target.offset + targetUpto]) +
- // " vs termLabel=" + (char) (term.byteAt(targetUpto)) + ")" + " arc.output=" + arc.output
+ // " vs termLabel=" + (char) (term.byteAt(targetUpto)) + ")" + " node.output=" +
+ // node.output
// + " output=" + output);
// }
if (cmp != 0) {
break;
}
- arc = arcs[1 + targetUpto];
- assert arc.label() == (target.bytes[target.offset + targetUpto] & 0xFF)
- : "arc.label="
- + (char) arc.label()
+ node = nodes[1 + targetUpto];
+ assert node.label == (target.bytes[target.offset + targetUpto] & 0xFF)
+ : "node.label="
+ + (char) node.label
+ " targetLabel="
+ (char) (target.bytes[target.offset + targetUpto] & 0xFF);
- outputAccumulator.push(arc.output());
- if (arc.isFinal()) {
+ if (node.hasOutput()) {
lastFrame = stack[1 + lastFrame.ord];
}
targetUpto++;
@@ -677,7 +598,7 @@ final class SegmentTermsEnum extends BaseTermsEnum {
if (cmp == 0) {
// Second compare the rest of the term, but
- // don't save arc/output/frame:
+ // don't save node/output/frame:
cmp =
Arrays.compareUnsigned(
term.bytes(),
@@ -728,25 +649,20 @@ final class SegmentTermsEnum extends BaseTermsEnum {
} else {
targetBeforeCurrentLength = -1;
- arc = fr.index.getFirstArc(arcs[0]);
+ node = nodes[0] = trieReader.root;
// Empty string prefix must have an output (block) in the index!
- assert arc.isFinal();
- assert arc.output() != null;
+ assert node.hasOutput();
// if (DEBUG) {
// System.out.println(" no seek state; push root frame");
// }
- outputAccumulator.push(arc.output());
-
currentFrame = staticFrame;
// term.length = 0;
targetUpto = 0;
- outputAccumulator.push(arc.nextFinalOutput());
- currentFrame = pushFrame(arc, 0);
- outputAccumulator.pop(arc.nextFinalOutput());
+ currentFrame = pushFrame(node, 0);
}
// if (DEBUG) {
@@ -761,10 +677,10 @@ final class SegmentTermsEnum extends BaseTermsEnum {
final int targetLabel = target.bytes[target.offset + targetUpto] & 0xFF;
- final FST.Arc<BytesRef> nextArc =
- fr.index.findTargetArc(targetLabel, arc, getArc(1 + targetUpto), fstReader);
+ final TrieReader.Node nextNode =
+ trieReader.lookupChild(targetLabel, node, getNode(1 + targetUpto));
- if (nextArc == null) {
+ if (nextNode == null) {
// Index is exhausted
// if (DEBUG) {
@@ -805,24 +721,19 @@ final class SegmentTermsEnum extends BaseTermsEnum {
return result;
}
} else {
- // Follow this arc
+ // Follow this node
term.setByteAt(targetUpto, (byte) targetLabel);
- arc = nextArc;
- // Aggregate output as we go:
- assert arc.output() != null;
- outputAccumulator.push(arc.output());
+ node = nextNode;
// if (DEBUG) {
// System.out.println(" index: follow label=" + (target.bytes[target.offset +
- // targetUpto]&0xff) + " arc.output=" + arc.output + " arc.nfo=" + arc.nextFinalOutput);
+ // targetUpto]&0xff) + " node.output=" + node.output + " node.nfo=" + node.nextFinalOutput);
// }
targetUpto++;
- if (arc.isFinal()) {
- // if (DEBUG) System.out.println(" arc is final!");
- outputAccumulator.push(arc.nextFinalOutput());
- currentFrame = pushFrame(arc, targetUpto);
- outputAccumulator.pop(arc.nextFinalOutput());
+ if (node.hasOutput()) {
+ // if (DEBUG) System.out.println(" node is final!");
+ currentFrame = pushFrame(node, targetUpto);
// if (DEBUG) System.out.println(" curFrame.ord=" + currentFrame.ord + " hasTerms=" +
// currentFrame.hasTerms);
}
@@ -887,10 +798,6 @@ final class SegmentTermsEnum extends BaseTermsEnum {
+ f.hasTerms
+ " isFloor="
+ f.isFloor
- + " code="
- + ((f.fp << Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)
- + (f.hasTerms ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- + (f.isFloor ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0))
+ " isLastInFloor="
+ f.isLastInFloor
+ " mdUpto="
@@ -917,10 +824,6 @@ final class SegmentTermsEnum extends BaseTermsEnum {
+ f.hasTerms
+ " isFloor="
+ f.isFloor
- + " code="
- + ((f.fp << Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)
- + (f.hasTerms ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- + (f.isFloor ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0))
+ " lastSubFP="
+ f.lastSubFP
+ " isLastInFloor="
@@ -930,40 +833,51 @@ final class SegmentTermsEnum extends BaseTermsEnum {
+ " tbOrd="
+ f.getTermBlockOrd());
}
- if (fr.index != null) {
- assert !isSeekFrame || f.arc != null : "isSeekFrame=" + isSeekFrame + " f.arc=" + f.arc;
- if (f.prefixLength > 0
- && isSeekFrame
- && f.arc.label() != (term.byteAt(f.prefixLength - 1) & 0xFF)) {
- out.println(
- " broken seek state: arc.label="
- + (char) f.arc.label()
- + " vs term byte="
- + (char) (term.byteAt(f.prefixLength - 1) & 0xFF));
- throw new RuntimeException("seek state is broken");
+ assert !isSeekFrame || f.node != null : "isSeekFrame=" + isSeekFrame + " f.node=" + f.node;
+ if (f.prefixLength > 0
+ && isSeekFrame
+ && f.node.label != (term.byteAt(f.prefixLength - 1) & 0xFF)) {
+ out.println(
+ " broken seek state: node.label="
+ + (char) f.node.label
+ + " vs term byte="
+ + (char) (term.byteAt(f.prefixLength - 1) & 0xFF));
+ throw new RuntimeException("seek state is broken");
+ }
+
+ TrieReader.Node node = trieReader.root;
+ TrieReader.Node child = new TrieReader.Node();
+ for (int i = 0; i < prefix.length; i++) {
+ TrieReader.Node found =
+ trieReader.lookupChild(prefix.bytes[i + prefix.offset] & 0xFF, node, child);
+ if (found == null) {
+ throw new RuntimeException("seek state is broken, prefix not exist in index");
}
- BytesRef output = Util.get(fr.index, prefix);
- if (output == null) {
- out.println(" broken seek state: prefix is not final in index");
+ node = child;
+ child = new TrieReader.Node();
+ }
+ if (!node.hasOutput()) {
+ out.println(" broken seek state: prefix is not final in index");
+ throw new RuntimeException("seek state is broken");
+ } else if (isSeekFrame && !f.isFloor) {
+ if (f.fp != node.outputFp || f.hasTerms != node.hasTerms || f.isFloor != node.isFloor()) {
+ out.println(
+ " broken seek state: output fp="
+ + node.outputFp
+ + ", hasTerms="
+ + node.hasTerms
+ + ", isFloor="
+ + node.isFloor()
+ + " doesn't match frame fp="
+ + f.fp
+ + ", hasTerms="
+ + f.hasTerms
+ + ", isFloor="
+ + f.isFloor);
throw new RuntimeException("seek state is broken");
- } else if (isSeekFrame && !f.isFloor) {
- final ByteArrayDataInput reader =
- new ByteArrayDataInput(output.bytes, output.offset, output.length);
- final long codeOrig = fr.readVLongOutput(reader);
- final long code =
- (f.fp << Lucene90BlockTreeTermsReader.OUTPUT_FLAGS_NUM_BITS)
- | (f.hasTerms ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_HAS_TERMS : 0)
- | (f.isFloor ? Lucene90BlockTreeTermsReader.OUTPUT_FLAG_IS_FLOOR : 0);
- if (codeOrig != code) {
- out.println(
- " broken seek state: output code="
- + codeOrig
- + " doesn't match frame code="
- + code);
- throw new RuntimeException("seek state is broken");
- }
}
}
+
if (f == currentFrame) {
break;
}
@@ -982,15 +896,8 @@ final class SegmentTermsEnum extends BaseTermsEnum {
public BytesRef next() throws IOException {
if (in == null) {
// Fresh TermsEnum; seek to first term:
- final FST.Arc<BytesRef> arc;
- if (fr.index != null) {
- arc = fr.index.getFirstArc(arcs[0]);
- // Empty string prefix must have an output in the index!
- assert arc.isFinal();
- } else {
- arc = null;
- }
- currentFrame = pushFrame(arc, fr.rootCode, 0);
+ final TrieReader.Node node = nodes[0] = trieReader.root;
+ currentFrame = pushFrame(node, 0);
currentFrame.loadBlock();
}
@@ -1160,81 +1067,4 @@ final class SegmentTermsEnum extends BaseTermsEnum {
public long ord() {
throw new UnsupportedOperationException();
}
-
- static class OutputAccumulator extends DataInput {
-
- BytesRef[] outputs = new BytesRef[16];
- BytesRef current;
- int num;
- int outputIndex;
- int index;
-
- void push(BytesRef output) {
- if (output != Lucene90BlockTreeTermsReader.NO_OUTPUT) {
- assert output.length > 0;
- outputs = ArrayUtil.grow(outputs, num + 1);
- outputs[num++] = output;
- }
- }
-
- void pop(BytesRef output) {
- if (output != Lucene90BlockTreeTermsReader.NO_OUTPUT) {
- assert num > 0;
- assert outputs[num - 1] == output;
- num--;
- }
- }
-
- void pop(int cnt) {
- assert num >= cnt;
- num -= cnt;
- }
-
- int outputCount() {
- return num;
- }
-
- void reset() {
- num = 0;
- }
-
- void prepareRead() {
- index = 0;
- outputIndex = 0;
- current = outputs[0];
- }
-
- /**
- * Set the last arc as the source of the floorData. This won't change the reading position of
- * this {@link OutputAccumulator}
- */
- void setFloorData(ByteArrayDataInput floorData) {
- assert outputIndex == num - 1
- : "floor data should be stored in last arc, get outputIndex: "
- + outputIndex
- + ", num: "
- + num;
- BytesRef output = outputs[outputIndex];
- floorData.reset(output.bytes, output.offset + index, output.length - index);
- }
-
- @Override
- public byte readByte() throws IOException {
- if (index >= current.length) {
- current = outputs[++outputIndex];
- index = 0;
- }
- return current.bytes[current.offset + index++];
- }
-
- @Override
- public void readBytes(byte[] b, int offset, int len) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void skipBytes(long numBytes) throws IOException {
- throw new UnsupportedOperationException();
- }
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/SegmentTermsEnumFrame.java
similarity index 96%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/SegmentTermsEnumFrame.java
index 85d23a489fe..1f3fcfcd05c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/SegmentTermsEnumFrame.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import java.io.IOException;
import java.util.Arrays;
@@ -23,9 +23,9 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.TermsEnum.SeekStatus;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.fst.FST;
final class SegmentTermsEnumFrame {
// Our index in stack[]:
@@ -35,7 +35,7 @@ final class SegmentTermsEnumFrame {
boolean hasTermsOrig;
boolean isFloor;
- FST.Arc<BytesRef> arc;
+ TrieReader.Node node;
// static boolean DEBUG = BlockTreeTermsWriter.DEBUG;
@@ -55,8 +55,10 @@ final class SegmentTermsEnumFrame {
int statsSingletonRunLength = 0;
final ByteArrayDataInput statsReader = new ByteArrayDataInput();
- int rewindPos;
- final ByteArrayDataInput floorDataReader = new ByteArrayDataInput();
+ long rewindPos;
+
+ long floorDataPos;
+ IndexInput floorDataReader;
// Length of prefix shared by all terms in this block
int prefixLength;
@@ -107,11 +109,12 @@ final class SegmentTermsEnumFrame {
suffixLengthsReader = new ByteArrayDataInput();
}
- public void setFloorData(SegmentTermsEnum.OutputAccumulator outputAccumulator) {
- outputAccumulator.setFloorData(floorDataReader);
- rewindPos = floorDataReader.getPosition();
- numFollowFloorBlocks = floorDataReader.readVInt();
- nextFloorLabel = floorDataReader.readByte() & 0xff;
+ public void setFloorData(IndexInput in) throws IOException {
+ floorDataReader = in;
+ rewindPos = in.getFilePointer();
+ numFollowFloorBlocks = in.readVInt();
+ nextFloorLabel = in.readByte() & 0xff;
+ floorDataPos = in.getFilePointer();
// if (DEBUG) {
// System.out.println(" setFloorData fpOrig=" + fpOrig + " bytes=" + new
// BytesRef(source.bytes, source.offset + in.getPosition(), numBytes) + " numFollowFloorBlocks="
@@ -127,7 +130,7 @@ final class SegmentTermsEnumFrame {
// if (DEBUG) {
// System.out.println(" loadNextFloorBlock fp=" + fp + " fpEnd=" + fpEnd);
// }
- assert arc == null || isFloor : "arc=" + arc + " isFloor=" + isFloor;
+ assert node == null || isFloor : "node=" + node + " isFloor=" + isFloor;
fp = fpEnd;
nextEnt = -1;
loadBlock();
@@ -176,8 +179,8 @@ final class SegmentTermsEnumFrame {
assert entCount > 0;
isLastInFloor = (code & 1) != 0;
- assert arc == null || (isLastInFloor || isFloor)
- : "fp=" + fp + " arc=" + arc + " isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;
+ assert node == null || (isLastInFloor || isFloor)
+ : "fp=" + fp + " node=" + node + " isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;
// TODO: if suffixes were stored in random-access
// array structure, then we could do binary search
@@ -215,7 +218,7 @@ final class SegmentTermsEnumFrame {
totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP;
/*if (DEBUG) {
- if (arc == null) {
+ if (node == null) {
System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);
} else {
System.out.println(" loadBlock (seek) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " hasTerms?=" + hasTerms + " isFloor?=" + isFloor + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock);
@@ -254,17 +257,18 @@ final class SegmentTermsEnumFrame {
// }
}
- void rewind() {
+ void rewind() throws IOException {
// Force reload:
fp = fpOrig;
nextEnt = -1;
hasTerms = hasTermsOrig;
if (isFloor) {
- floorDataReader.setPosition(rewindPos);
+ floorDataReader.seek(rewindPos);
numFollowFloorBlocks = floorDataReader.readVInt();
assert numFollowFloorBlocks > 0;
nextFloorLabel = floorDataReader.readByte() & 0xff;
+ floorDataPos = floorDataReader.getFilePointer();
}
/*
@@ -331,7 +335,7 @@ final class SegmentTermsEnumFrame {
// + entCount + " fp=" + suffixesReader.getPosition());
while (true) {
if (nextEnt == entCount) {
- assert arc == null || (isFloor && isLastInFloor == false)
+ assert node == null || (isFloor && isLastInFloor == false)
: "isFloor=" + isFloor + " isLastInFloor=" + isLastInFloor;
loadNextFloorBlock();
if (isLeafBlock) {
@@ -373,7 +377,7 @@ final class SegmentTermsEnumFrame {
// TODO: make this array'd so we can do bin search?
// likely not worth it? need to measure how many
// floor blocks we "typically" get
- public void scanToFloorFrame(BytesRef target) {
+ public void scanToFloorFrame(BytesRef target) throws IOException {
if (!isFloor || target.length <= prefixLength) {
// if (DEBUG) {
@@ -401,6 +405,7 @@ final class SegmentTermsEnumFrame {
assert numFollowFloorBlocks != 0;
long newFP = fpOrig;
+ floorDataReader.seek(floorDataPos);
while (true) {
final long code = floorDataReader.readVLong();
newFP = fpOrig + (code >>> 1);
@@ -430,7 +435,7 @@ final class SegmentTermsEnumFrame {
}
}
}
-
+ floorDataPos = floorDataReader.getFilePointer();
if (newFP != fp) {
// Force re-load of the block:
// if (DEBUG) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Stats.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Stats.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Stats.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Stats.java
index ceeef4a8687..7d754a519ce 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Stats.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/Stats.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -181,7 +181,7 @@ public class Stats {
final ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
PrintStream out = new PrintStream(bos, false, UTF_8);
- out.println(" index FST:");
+ out.println(" index trie:");
out.println(" " + indexNumBytes + " bytes");
out.println(" terms:");
out.println(" " + totalTermCount + " terms");
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/TrieBuilder.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/TrieBuilder.java
new file mode 100644
index 00000000000..63215d18e6e
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/TrieBuilder.java
@@ -0,0 +1,641 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene103.blocktree;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.function.BiConsumer;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.store.RandomAccessInput;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+
+/**
+ * A builder to build prefix tree (trie) as the index of block tree, and can be saved to disk.
+ *
+ * <p>TODO make this trie builder a more memory efficient structure.
+ */
+class TrieBuilder {
+
+ static final int SIGN_NO_CHILDREN = 0x00;
+ static final int SIGN_SINGLE_CHILD_WITH_OUTPUT = 0x01;
+ static final int SIGN_SINGLE_CHILD_WITHOUT_OUTPUT = 0x02;
+ static final int SIGN_MULTI_CHILDREN = 0x03;
+
+ static final int LEAF_NODE_HAS_TERMS = 1 << 5;
+ static final int LEAF_NODE_HAS_FLOOR = 1 << 6;
+ static final long NON_LEAF_NODE_HAS_TERMS = 1L << 1;
+ static final long NON_LEAF_NODE_HAS_FLOOR = 1L << 0;
+
+ /**
+ * The output describing the term block the prefix point to.
+ *
+ * @param fp the file pointer to the on-disk terms block which a trie node points to.
+ * @param hasTerms false if this on-disk block consists entirely of pointers to child blocks.
+ * @param floorData will be non-null when a large block of terms sharing a single trie prefix is
+ * split into multiple on-disk blocks.
+ */
+ record Output(long fp, boolean hasTerms, BytesRef floorData) {}
+
+ private enum Status {
+ BUILDING,
+ SAVED,
+ DESTROYED
+ }
+
+ private static class Node {
+
+ // The utf8 digit that leads to this Node, 0 for root node
+ private final int label;
+ // The output of this node.
+ private Output output;
+ // The number of children of this node.
+ private int childrenNum;
+ // Pointers to relative nodes
+ private Node next;
+ private Node firstChild;
+ private Node lastChild;
+
+ // Vars used during saving:
+
+ // The file pointer point to where the node saved. -1 means the node has not been saved.
+ private long fp = -1;
+ // The latest child that have been saved. null means no child has been saved.
+ private Node savedTo;
+
+ Node(int label, Output output) {
+ this.label = label;
+ this.output = output;
+ }
+ }
+
+ private final Node root = new Node(0, null);
+ private final BytesRef minKey;
+ private BytesRef maxKey;
+ private Status status = Status.BUILDING;
+
+ static TrieBuilder bytesRefToTrie(BytesRef k, Output v) {
+ return new TrieBuilder(k, v);
+ }
+
+ private TrieBuilder(BytesRef k, Output v) {
+ minKey = maxKey = BytesRef.deepCopyOf(k);
+ if (k.length == 0) {
+ root.output = v;
+ return;
+ }
+ Node parent = root;
+ for (int i = 0; i < k.length; i++) {
+ int b = k.bytes[i + k.offset] & 0xFF;
+ Output output = i == k.length - 1 ? v : null;
+ Node node = new Node(b, output);
+ parent.firstChild = parent.lastChild = node;
+ parent.childrenNum = 1;
+ parent = node;
+ }
+ }
+
+ /**
+ * Append all (K, V) pairs from the given trie into this one. The given trie builder need to
+ * ensure its keys greater or equals than max key of this one.
+ *
+ * <p>Note: the given trie will be destroyed after appending.
+ */
+ void append(TrieBuilder trieBuilder) {
+ if (status != Status.BUILDING || trieBuilder.status != Status.BUILDING) {
+ throw new IllegalStateException(
+ "tries have wrong status, got this: " + status + ", append: " + trieBuilder.status);
+ }
+ assert this.maxKey.compareTo(trieBuilder.minKey) < 0;
+
+ int mismatch =
+ Arrays.mismatch(
+ this.maxKey.bytes,
+ this.maxKey.offset,
+ this.maxKey.offset + this.maxKey.length,
+ trieBuilder.minKey.bytes,
+ trieBuilder.minKey.offset,
+ trieBuilder.minKey.offset + trieBuilder.minKey.length);
+ Node a = this.root;
+ Node b = trieBuilder.root;
+
+ for (int i = 0; i < mismatch; i++) {
+ final Node aLast = a.lastChild;
+ final Node bFirst = b.firstChild;
+ assert aLast.label == bFirst.label;
+
+ if (b.childrenNum > 1) {
+ aLast.next = bFirst.next;
+ a.childrenNum += b.childrenNum - 1;
+ a.lastChild = b.lastChild;
+ assert assertChildrenLabelInOrder(a);
+ }
+
+ a = aLast;
+ b = bFirst;
+ }
+
+ assert b.childrenNum > 0;
+ if (a.childrenNum == 0) {
+ a.firstChild = b.firstChild;
+ a.lastChild = b.lastChild;
+ a.childrenNum = b.childrenNum;
+ } else {
+ assert a.lastChild.label < b.firstChild.label;
+ a.lastChild.next = b.firstChild;
+ a.lastChild = b.lastChild;
+ a.childrenNum += b.childrenNum;
+ }
+ assert assertChildrenLabelInOrder(a);
+
+ this.maxKey = trieBuilder.maxKey;
+ trieBuilder.status = Status.DESTROYED;
+ }
+
+ Output getEmptyOutput() {
+ return root.output;
+ }
+
+ /**
+ * Used for tests only. The recursive impl need to be avoided if someone plans to use for
+ * production one day.
+ */
+ void visit(BiConsumer<BytesRef, Output> consumer) {
+ assert status == Status.BUILDING;
+ if (root.output != null) {
+ consumer.accept(new BytesRef(), root.output);
+ }
+ visit(root.firstChild, new BytesRefBuilder(), consumer);
+ }
+
+ private void visit(Node first, BytesRefBuilder key, BiConsumer<BytesRef, Output> consumer) {
+ while (first != null) {
+ key.append((byte) first.label);
+ if (first.output != null) {
+ consumer.accept(key.toBytesRef(), first.output);
+ }
+ visit(first.firstChild, key, consumer);
+ key.setLength(key.length() - 1);
+ first = first.next;
+ }
+ }
+
+ void save(DataOutput meta, IndexOutput index) throws IOException {
+ if (status != Status.BUILDING) {
+ throw new IllegalStateException("only unsaved trie can be saved, got: " + status);
+ }
+ meta.writeVLong(index.getFilePointer());
+ saveNodes(index);
+ meta.writeVLong(root.fp);
+ index.writeLong(0L); // additional 8 bytes for over-reading
+ meta.writeVLong(index.getFilePointer());
+ status = Status.SAVED;
+ }
+
+ void saveNodes(IndexOutput index) throws IOException {
+ final long startFP = index.getFilePointer();
+ Deque<Node> stack = new ArrayDeque<>();
+ stack.push(root);
+
+ // Visit and save nodes of this trie in a post-order depth-first traversal.
+ while (stack.isEmpty() == false) {
+ Node node = stack.peek();
+ assert node.fp == -1;
+ assert assertChildrenLabelInOrder(node);
+
+ final int childrenNum = node.childrenNum;
+
+ if (childrenNum == 0) { // leaf node
+ assert node.output != null : "leaf nodes should have output.";
+
+ node.fp = index.getFilePointer() - startFP;
+ stack.pop();
+
+ // [n bytes] floor data
+ // [n bytes] output fp
+ // [1bit] x | [1bit] has floor | [1bit] has terms | [3bit] output fp bytes | [2bit] sign
+
+ Output output = node.output;
+ int outputFpBytes = bytesRequiredVLong(output.fp);
+ int header =
+ SIGN_NO_CHILDREN
+ | ((outputFpBytes - 1) << 2)
+ | (output.hasTerms ? LEAF_NODE_HAS_TERMS : 0)
+ | (output.floorData != null ? LEAF_NODE_HAS_FLOOR : 0);
+ index.writeByte(((byte) header));
+ writeLongNBytes(output.fp, outputFpBytes, index);
+ if (output.floorData != null) {
+ index.writeBytes(
+ output.floorData.bytes, output.floorData.offset, output.floorData.length);
+ }
+ continue;
+ }
+
+ // If there are any children have not been saved, push the first one into stack and continue.
+ // We want to ensure saving children before parent.
+
+ if (node.savedTo == null) {
+ node.savedTo = node.firstChild;
+ stack.push(node.savedTo);
+ continue;
+ }
+ if (node.savedTo.next != null) {
+ assert node.savedTo.fp >= 0;
+ node.savedTo = node.savedTo.next;
+ stack.push(node.savedTo);
+ continue;
+ }
+
+ // All children have been written, now it's time to write the parent!
+
+ assert assertNonLeafNodePreparingSaving(node);
+ node.fp = index.getFilePointer() - startFP;
+ stack.pop();
+
+ if (childrenNum == 1) {
+
+ // [n bytes] floor data
+ // [n bytes] encoded output fp | [n bytes] child fp | [1 byte] label
+ // [3bit] encoded output fp bytes | [3bit] child fp bytes | [2bit] sign
+
+ long childDeltaFp = node.fp - node.firstChild.fp;
+ assert childDeltaFp > 0 : "parent node is always written after children: " + childDeltaFp;
+ int childFpBytes = bytesRequiredVLong(childDeltaFp);
+ int encodedOutputFpBytes =
+ node.output == null ? 0 : bytesRequiredVLong(node.output.fp << 2);
+
+ // TODO if we have only one child and no output, we can store child labels in this node.
+ // E.g. for a single term trie [foobar], we can save only two nodes [fooba] and [r]
+
+ int sign =
+ node.output != null ? SIGN_SINGLE_CHILD_WITH_OUTPUT : SIGN_SINGLE_CHILD_WITHOUT_OUTPUT;
+ int header = sign | ((childFpBytes - 1) << 2) | ((encodedOutputFpBytes - 1) << 5);
+ index.writeByte((byte) header);
+ index.writeByte((byte) node.firstChild.label);
+ writeLongNBytes(childDeltaFp, childFpBytes, index);
+
+ if (node.output != null) {
+ Output output = node.output;
+ long encodedFp = encodeFP(output);
+ writeLongNBytes(encodedFp, encodedOutputFpBytes, index);
+ if (output.floorData != null) {
+ index.writeBytes(
+ output.floorData.bytes, output.floorData.offset, output.floorData.length);
+ }
+ }
+ } else {
+
+ // [n bytes] floor data
+ // [n bytes] children fps | [n bytes] strategy data
+ // [1 byte] children count (if floor data) | [n bytes] encoded output fp | [1 byte] label
+ // [5bit] strategy bytes | 2bit children strategy | [3bit] encoded output fp bytes
+ // [1bit] has output | [3bit] children fp bytes | [2bit] sign
+
+ final int minLabel = node.firstChild.label;
+ final int maxLabel = node.lastChild.label;
+ assert maxLabel > minLabel;
+ ChildSaveStrategy childSaveStrategy =
+ ChildSaveStrategy.choose(minLabel, maxLabel, childrenNum);
+ int strategyBytes = childSaveStrategy.needBytes(minLabel, maxLabel, childrenNum);
+ assert strategyBytes > 0 && strategyBytes <= 32;
+
+ // children fps are in order, so the first child's fp is min, then delta is max.
+ long maxChildDeltaFp = node.fp - node.firstChild.fp;
+ assert maxChildDeltaFp > 0 : "parent always written after all children";
+
+ int childrenFpBytes = bytesRequiredVLong(maxChildDeltaFp);
+ int encodedOutputFpBytes =
+ node.output == null ? 1 : bytesRequiredVLong(node.output.fp << 2);
+ int header =
+ SIGN_MULTI_CHILDREN
+ | ((childrenFpBytes - 1) << 2)
+ | ((node.output != null ? 1 : 0) << 5)
+ | ((encodedOutputFpBytes - 1) << 6)
+ | (childSaveStrategy.code << 9)
+ | ((strategyBytes - 1) << 11)
+ | (minLabel << 16);
+
+ writeLongNBytes(header, 3, index);
+
+ if (node.output != null) {
+ Output output = node.output;
+ long encodedFp = encodeFP(output);
+ writeLongNBytes(encodedFp, encodedOutputFpBytes, index);
+ if (output.floorData != null) {
+ // We need this childrenNum to compute where the floor data start.
+ index.writeByte((byte) (childrenNum - 1));
+ }
+ }
+
+ long strategyStartFp = index.getFilePointer();
+ childSaveStrategy.save(node, childrenNum, strategyBytes, index);
+ assert index.getFilePointer() == strategyStartFp + strategyBytes
+ : childSaveStrategy.name()
+ + " strategy bytes compute error, computed: "
+ + strategyBytes
+ + " actual: "
+ + (index.getFilePointer() - strategyStartFp);
+
+ for (Node child = node.firstChild; child != null; child = child.next) {
+ assert node.fp > child.fp : "parent always written after all children";
+ writeLongNBytes(node.fp - child.fp, childrenFpBytes, index);
+ }
+
+ if (node.output != null && node.output.floorData != null) {
+ BytesRef floorData = node.output.floorData;
+ index.writeBytes(floorData.bytes, floorData.offset, floorData.length);
+ }
+ }
+ }
+ }
+
+ private long encodeFP(Output output) {
+ assert output.fp < 1L << 62;
+ return (output.floorData != null ? NON_LEAF_NODE_HAS_FLOOR : 0)
+ | (output.hasTerms ? NON_LEAF_NODE_HAS_TERMS : 0)
+ | (output.fp << 2);
+ }
+
+ private static int bytesRequiredVLong(long v) {
+ return Long.BYTES - (Long.numberOfLeadingZeros(v | 1) >>> 3);
+ }
+
+ /**
+ * Write the first (LSB order) n bytes of the given long v into the DataOutput.
+ *
+ * <p>This differs from writeVLong because it can write more bytes than would be needed for vLong
+ * when the incoming int n is larger.
+ */
+ private static void writeLongNBytes(long v, int n, DataOutput out) throws IOException {
+ for (int i = 0; i < n; i++) {
+ // Note that we sometimes write trailing 0 bytes here, when the incoming int n is bigger than
+ // would be required for a "normal" vLong
+ out.writeByte((byte) v);
+ v >>>= 8;
+ }
+ assert v == 0;
+ }
+
+ private static boolean assertChildrenLabelInOrder(Node node) {
+ if (node.childrenNum == 0) {
+ assert node.firstChild == null;
+ assert node.lastChild == null;
+ } else if (node.childrenNum == 1) {
+ assert node.firstChild == node.lastChild;
+ assert node.firstChild.next == null;
+ } else if (node.childrenNum > 1) {
+ int n = 0;
+ for (Node child = node.firstChild; child != null; child = child.next) {
+ n++;
+ assert child.next == null || child.label < child.next.label
+ : " the label of children nodes should always be in strictly increasing order.";
+ }
+ assert node.childrenNum == n;
+ }
+ return true;
+ }
+
+ private static boolean assertNonLeafNodePreparingSaving(Node node) {
+ assert assertChildrenLabelInOrder(node);
+ assert node.childrenNum != 0;
+ if (node.childrenNum == 1) {
+ assert node.firstChild == node.lastChild;
+ assert node.firstChild.next == null;
+ assert node.savedTo == node.firstChild;
+ assert node.firstChild.fp >= 0;
+ } else {
+ int n = 0;
+ for (Node child = node.firstChild; child != null; child = child.next) {
+ n++;
+ assert child.fp >= 0;
+ assert child.next == null || child.fp < child.next.fp
+ : " the fp or children nodes should always be in order.";
+ }
+ assert node.childrenNum == n;
+ assert node.lastChild == node.savedTo;
+ assert node.savedTo.next == null;
+ }
+ return true;
+ }
+
+ enum ChildSaveStrategy {
+
+ /**
+ * Store children labels in a bitset, this is likely the most efficient storage as we can
+ * compute position with bitCount instruction, so we give it the highest priority.
+ */
+ BITS(2) {
+ @Override
+ int needBytes(int minLabel, int maxLabel, int labelCnt) {
+ int byteDistance = maxLabel - minLabel + 1;
+ return (byteDistance + 7) >>> 3;
+ }
+
+ @Override
+ void save(Node parent, int labelCnt, int strategyBytes, IndexOutput output)
+ throws IOException {
+ byte presenceBits = 1; // The first arc is always present.
+ int presenceIndex = 0;
+ int previousLabel = parent.firstChild.label;
+ for (Node child = parent.firstChild.next; child != null; child = child.next) {
+ int label = child.label;
+ assert label > previousLabel;
+ presenceIndex += label - previousLabel;
+ while (presenceIndex >= Byte.SIZE) {
+ output.writeByte(presenceBits);
+ presenceBits = 0;
+ presenceIndex -= Byte.SIZE;
+ }
+ // Set the bit at presenceIndex to flag that the corresponding arc is present.
+ presenceBits |= 1 << presenceIndex;
+ previousLabel = label;
+ }
+ assert presenceIndex == (parent.lastChild.label - parent.firstChild.label) % 8;
+ assert presenceBits != 0; // The last byte is not 0.
+ assert (presenceBits & (1 << presenceIndex)) != 0; // The last arc is always present.
+ output.writeByte(presenceBits);
+ }
+
+ @Override
+ int lookup(
+ int targetLabel, RandomAccessInput in, long offset, int strategyBytes, int minLabel)
+ throws IOException {
+ int bitIndex = targetLabel - minLabel;
+ if (bitIndex >= (strategyBytes << 3)) {
+ return -1;
+ }
+ int wordIndex = bitIndex >>> 6;
+ long wordFp = offset + (wordIndex << 3);
+ long word = in.readLong(wordFp);
+ long mask = 1L << bitIndex;
+ if ((word & mask) == 0) {
+ return -1;
+ }
+ int pos = 0;
+ for (long fp = offset; fp < wordFp; fp += 8L) {
+ pos += Long.bitCount(in.readLong(fp));
+ }
+ pos += Long.bitCount(word & (mask - 1));
+ return pos;
+ }
+ },
+
+ /**
+ * Store labels in an array and lookup with binary search.
+ *
+ * <p>TODO: Can we use VectorAPI to speed up the lookup? we can check 64 labels once on AVX512!
+ */
+ ARRAY(1) {
+ @Override
+ int needBytes(int minLabel, int maxLabel, int labelCnt) {
+ return labelCnt - 1; // min label saved
+ }
+
+ @Override
+ void save(Node parent, int labelCnt, int strategyBytes, IndexOutput output)
+ throws IOException {
+ for (Node child = parent.firstChild.next; child != null; child = child.next) {
+ output.writeByte((byte) child.label);
+ }
+ }
+
+ @Override
+ int lookup(
+ int targetLabel, RandomAccessInput in, long offset, int strategyBytes, int minLabel)
+ throws IOException {
+ int low = 0;
+ int high = strategyBytes - 1;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ int midLabel = in.readByte(offset + mid) & 0xFF;
+ if (midLabel < targetLabel) {
+ low = mid + 1;
+ } else if (midLabel > targetLabel) {
+ high = mid - 1;
+ } else {
+ return mid + 1; // min label not included, plus 1
+ }
+ }
+ return -1;
+ }
+ },
+
+ /**
+ * Store labels that not existing within the range. E.g. store 10(max label) and 3, 5(absent
+ * label) for [1, 2, 4, 6, 7, 8, 9, 10].
+ *
+ * <p>TODO: Can we use VectorAPI to speed up the lookup? we can check 64 labels once on AVX512!
+ */
+ REVERSE_ARRAY(0) {
+
+ @Override
+ int needBytes(int minLabel, int maxLabel, int labelCnt) {
+ int byteDistance = maxLabel - minLabel + 1;
+ return byteDistance - labelCnt + 1;
+ }
+
+ @Override
+ void save(Node parent, int labelCnt, int strategyBytes, IndexOutput output)
+ throws IOException {
+ output.writeByte((byte) parent.lastChild.label);
+ int lastLabel = parent.firstChild.label;
+ for (Node child = parent.firstChild.next; child != null; child = child.next) {
+ while (++lastLabel < child.label) {
+ output.writeByte((byte) lastLabel);
+ }
+ }
+ }
+
+ @Override
+ int lookup(
+ int targetLabel, RandomAccessInput in, long offset, int strategyBytes, int minLabel)
+ throws IOException {
+ int maxLabel = in.readByte(offset++) & 0xFF;
+ if (targetLabel >= maxLabel) {
+ return targetLabel == maxLabel ? maxLabel - minLabel - strategyBytes + 1 : -1;
+ }
+ if (strategyBytes == 1) {
+ return targetLabel - minLabel;
+ }
+
+ int low = 0;
+ int high = strategyBytes - 2;
+ while (low <= high) {
+ int mid = (low + high) >>> 1;
+ int midLabel = in.readByte(offset + mid) & 0xFF;
+ if (midLabel < targetLabel) {
+ low = mid + 1;
+ } else if (midLabel > targetLabel) {
+ high = mid - 1;
+ } else {
+ return -1;
+ }
+ }
+ return targetLabel - minLabel - low;
+ }
+ };
+
+ private static final ChildSaveStrategy[] STRATEGIES_IN_PRIORITY_ORDER =
+ new ChildSaveStrategy[] {BITS, ARRAY, REVERSE_ARRAY};
+ private static final ChildSaveStrategy[] STRATEGIES_BY_CODE;
+
+ static {
+ STRATEGIES_BY_CODE = new ChildSaveStrategy[ChildSaveStrategy.values().length];
+ for (ChildSaveStrategy strategy : ChildSaveStrategy.values()) {
+ assert STRATEGIES_BY_CODE[strategy.code] == null;
+ STRATEGIES_BY_CODE[strategy.code] = strategy;
+ }
+ }
+
+ final int code;
+
+ ChildSaveStrategy(int code) {
+ this.code = code;
+ }
+
+ abstract int needBytes(int minLabel, int maxLabel, int labelCnt);
+
+ abstract void save(Node parent, int labelCnt, int strategyBytes, IndexOutput output)
+ throws IOException;
+
+ abstract int lookup(
+ int targetLabel, RandomAccessInput in, long offset, int strategyBytes, int minLabel)
+ throws IOException;
+
+ static ChildSaveStrategy byCode(int code) {
+ return STRATEGIES_BY_CODE[code];
+ }
+
+ static ChildSaveStrategy choose(int minLabel, int maxLabel, int labelCnt) {
+ ChildSaveStrategy childSaveStrategy = null;
+ int strategyBytes = Integer.MAX_VALUE;
+ for (ChildSaveStrategy strategy : ChildSaveStrategy.STRATEGIES_IN_PRIORITY_ORDER) {
+ int strategyCost = strategy.needBytes(minLabel, maxLabel, labelCnt);
+ if (strategyCost < strategyBytes) {
+ childSaveStrategy = strategy;
+ strategyBytes = strategyCost;
+ }
+ }
+ assert childSaveStrategy != null;
+ assert strategyBytes > 0 && strategyBytes <= 32;
+ return childSaveStrategy;
+ }
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/TrieReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/TrieReader.java
new file mode 100644
index 00000000000..367be262cc6
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/TrieReader.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene103.blocktree;
+
+import java.io.IOException;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.RandomAccessInput;
+
+class TrieReader {
+
+ private static final long NO_OUTPUT = -1;
+ private static final long NO_FLOOR_DATA = -1;
+ private static final long[] BYTES_MINUS_1_MASK =
+ new long[] {
+ 0xFFL,
+ 0xFFFFL,
+ 0xFFFFFFL,
+ 0xFFFFFFFFL,
+ 0xFFFFFFFFFFL,
+ 0xFFFFFFFFFFFFL,
+ 0xFFFFFFFFFFFFFFL,
+ 0xFFFFFFFFFFFFFFFFL
+ };
+
+ static class Node {
+
+ // single child
+ private long childDeltaFp;
+
+ // multi children
+ private long strategyFp;
+ private int childSaveStrategy;
+ private int strategyBytes;
+ private int childrenDeltaFpBytes;
+
+ // common
+ private int sign;
+ private long fp;
+ private int minChildrenLabel;
+ int label;
+ long outputFp;
+ boolean hasTerms;
+ long floorDataFp;
+
+ boolean hasOutput() {
+ return outputFp != NO_OUTPUT;
+ }
+
+ boolean isFloor() {
+ return floorDataFp != NO_FLOOR_DATA;
+ }
+
+ IndexInput floorData(TrieReader r) throws IOException {
+ assert isFloor();
+ r.input.seek(floorDataFp);
+ return r.input;
+ }
+ }
+
+ final RandomAccessInput access;
+ final IndexInput input;
+ final Node root;
+
+ TrieReader(IndexInput input, long rootFP) throws IOException {
+ this.access = input.randomAccessSlice(0, input.length());
+ this.input = input;
+ this.root = new Node();
+ load(root, rootFP);
+ }
+
+ private void load(Node node, long fp) throws IOException {
+ node.fp = fp;
+ long termFlagsLong = access.readLong(fp);
+ int termFlags = (int) termFlagsLong;
+ int sign = node.sign = termFlags & 0x03;
+
+ if (sign == TrieBuilder.SIGN_NO_CHILDREN) {
+ loadLeafNode(fp, termFlags, termFlagsLong, node);
+ } else if (sign == TrieBuilder.SIGN_MULTI_CHILDREN) {
+ loadMultiChildrenNode(fp, termFlags, termFlagsLong, node);
+ } else {
+ loadSingleChildNode(fp, sign, termFlags, termFlagsLong, node);
+ }
+ }
+
+ private void loadLeafNode(long fp, int term, long termLong, Node node) throws IOException {
+
+ // [n bytes] floor data
+ // [n bytes] output fp
+ // [1bit] x | [1bit] has floor | [1bit] has terms | [3bit] output fp bytes | [2bit] sign
+
+ int fpBytesMinus1 = (term >>> 2) & 0x07;
+ if (fpBytesMinus1 <= 6) {
+ node.outputFp = (termLong >>> 8) & BYTES_MINUS_1_MASK[fpBytesMinus1];
+ } else {
+ node.outputFp = access.readLong(fp + 1);
+ }
+ node.hasTerms = (term & TrieBuilder.LEAF_NODE_HAS_TERMS) != 0;
+ if ((term & TrieBuilder.LEAF_NODE_HAS_FLOOR) != 0) { // has floor
+ node.floorDataFp = fp + 2 + fpBytesMinus1;
+ } else {
+ node.floorDataFp = NO_FLOOR_DATA;
+ }
+ }
+
+ private void loadSingleChildNode(long fp, int sign, int term, long termLong, Node node)
+ throws IOException {
+
+ // [n bytes] floor data
+ // [n bytes] encoded output fp | [n bytes] child fp | [1 byte] label
+ // [3bit] encoded output fp bytes | [3bit] child fp bytes | [2bit] sign
+
+ int childDeltaFpBytesMinus1 = (term >>> 2) & 0x07;
+ long l = childDeltaFpBytesMinus1 <= 5 ? termLong >>> 16 : access.readLong(fp + 2);
+ node.childDeltaFp = l & BYTES_MINUS_1_MASK[childDeltaFpBytesMinus1];
+ node.minChildrenLabel = (term >>> 8) & 0xFF;
+
+ if (sign == TrieBuilder.SIGN_SINGLE_CHILD_WITHOUT_OUTPUT) {
+ node.outputFp = NO_OUTPUT;
+ } else { // has output
+ assert sign == TrieBuilder.SIGN_SINGLE_CHILD_WITH_OUTPUT;
+ int encodedOutputFpBytesMinus1 = (term >>> 5) & 0x07;
+ long offset = fp + childDeltaFpBytesMinus1 + 3;
+ long encodedFp = access.readLong(offset) & BYTES_MINUS_1_MASK[encodedOutputFpBytesMinus1];
+ node.outputFp = encodedFp >>> 2;
+ node.hasTerms = (encodedFp & TrieBuilder.NON_LEAF_NODE_HAS_TERMS) != 0;
+ if ((encodedFp & TrieBuilder.NON_LEAF_NODE_HAS_FLOOR) != 0) { // has floor
+ node.floorDataFp = offset + encodedOutputFpBytesMinus1 + 1;
+ } else {
+ node.floorDataFp = NO_FLOOR_DATA;
+ }
+ }
+ }
+
+ private void loadMultiChildrenNode(long fp, int term, long termLong, Node node)
+ throws IOException {
+
+ // [n bytes] floor data
+ // [n bytes] children fps | [n bytes] strategy data
+ // [1 byte] children count (if floor data) | [n bytes] encoded output fp | [1 byte] label
+ // [5bit] strategy bytes | 2bit children strategy | [3bit] encoded output fp bytes
+ // [1bit] has output | [3bit] children fp bytes | [2bit] sign
+
+ node.childrenDeltaFpBytes = ((term >>> 2) & 0x07) + 1;
+ node.childSaveStrategy = (term >>> 9) & 0x03;
+ node.strategyBytes = ((term >>> 11) & 0x1F) + 1;
+ node.minChildrenLabel = (term >>> 16) & 0xFF;
+
+ if ((term & 0x20) != 0) { // has output
+ int encodedOutputFpBytesMinus1 = (term >>> 6) & 0x07;
+ long l = encodedOutputFpBytesMinus1 <= 4 ? termLong >>> 24 : access.readLong(fp + 3);
+ long encodedFp = l & BYTES_MINUS_1_MASK[encodedOutputFpBytesMinus1];
+ node.outputFp = encodedFp >>> 2;
+ node.hasTerms = (encodedFp & TrieBuilder.NON_LEAF_NODE_HAS_TERMS) != 0;
+
+ if ((encodedFp & TrieBuilder.NON_LEAF_NODE_HAS_FLOOR) != 0) { // has floor
+ long offset = fp + 4 + encodedOutputFpBytesMinus1;
+ long childrenNum = (access.readByte(offset) & 0xFFL) + 1L;
+ node.strategyFp = offset + 1L;
+ node.floorDataFp =
+ node.strategyFp + node.strategyBytes + childrenNum * node.childrenDeltaFpBytes;
+ } else {
+ node.floorDataFp = NO_FLOOR_DATA;
+ node.strategyFp = fp + 4 + encodedOutputFpBytesMinus1;
+ }
+ } else {
+ node.outputFp = NO_OUTPUT;
+ node.strategyFp = fp + 3;
+ }
+ }
+
+ /** Overwrite (and return) the incoming Node child, or null if the targetLabel was not found. */
+ Node lookupChild(int targetLabel, Node parent, Node child) throws IOException {
+ final int sign = parent.sign;
+ if (sign == TrieBuilder.SIGN_NO_CHILDREN) {
+ return null;
+ }
+
+ if (sign != TrieBuilder.SIGN_MULTI_CHILDREN) {
+ // single child
+ if (targetLabel != parent.minChildrenLabel) {
+ return null;
+ }
+ child.label = targetLabel;
+ load(child, parent.fp - parent.childDeltaFp);
+ return child;
+ }
+
+ final long strategyBytesStartFp = parent.strategyFp;
+ final int minLabel = parent.minChildrenLabel;
+ final int strategyBytes = parent.strategyBytes;
+
+ int position = -1;
+ if (targetLabel == minLabel) {
+ position = 0;
+ } else if (targetLabel > minLabel) {
+ position =
+ TrieBuilder.ChildSaveStrategy.byCode(parent.childSaveStrategy)
+ .lookup(targetLabel, access, strategyBytesStartFp, strategyBytes, minLabel);
+ }
+
+ if (position < 0) {
+ return null;
+ }
+
+ final int bytesPerEntry = parent.childrenDeltaFpBytes;
+ final long pos = strategyBytesStartFp + strategyBytes + (long) bytesPerEntry * position;
+ final long fp = parent.fp - (access.readLong(pos) & BYTES_MINUS_1_MASK[bytesPerEntry - 1]);
+ child.label = targetLabel;
+ load(child, fp);
+
+ return child;
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/package-info.java
similarity index 86%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/package-info.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/package-info.java
index 27d57b60371..71c963ac0d3 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/blocktree/package-info.java
@@ -19,11 +19,11 @@
* BlockTree terms dictionary.
*
* <p>This terms dictionary organizes all terms into blocks according to shared prefix, such that
- * each block has enough terms, and then stores the prefix trie in memory as an FST as the index
+ * each block has enough terms, and then stores the prefix trie in memory as a trie as the index
* structure. It allows you to plug in your own {@link org.apache.lucene.codecs.PostingsWriterBase}
* to implement the postings.
*
- * <p>See {@link org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter} for the
+ * <p>See {@link org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter} for the
* file format.
*/
-package org.apache.lucene.codecs.lucene90.blocktree;
+package org.apache.lucene.codecs.lucene103.blocktree;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py
index b1b36db096a..f4f3fed630c 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForDeltaUtil.py
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForDeltaUtil.py
@@ -40,14 +40,14 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.packed.PackedInts;
-import static org.apache.lucene.codecs.lucene101.ForUtil.*;
+import static org.apache.lucene.codecs.lucene103.ForUtil.*;
/**
* Inspired from https://fulmicoton.com/posts/bitpacking/
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py
index 4640c9bd886..6b1a977a99d 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene101/gen_ForUtil.py
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/gen_ForUtil.py
@@ -40,7 +40,7 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.internal.vectorization.PostingDecodingUtil;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java
similarity index 96%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
copy to lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java
index 8f6fcb2ef5b..0c4bb5a1dde 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene102/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene103/package-info.java
@@ -16,7 +16,7 @@
*/
/**
- * Lucene 10.2 file format.
+ * Lucene 10.3 file format.
*
* <h2>Apache Lucene - Index File Formats</h2>
*
@@ -151,15 +151,15 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term dictionary}. A
+ * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term dictionary}. A
* dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Frequency data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Frequency data}. For
* each term in the dictionary, the numbers of all the documents that contain that term, and
* the frequency of the term in that document, unless frequencies are omitted ({@link
* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Proximity data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Proximity data}. For
* each term in the dictionary, the positions that the term occurs in each document. Note that
* this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
@@ -311,11 +311,10 @@
* </tr>
* <tr>
* <td>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}</td>
- * <td>.vec, .vem, .veq, .vex, .veb, .vemb</td>
+ * <td>.vec, .vem, .veq, vex</td>
* <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data,
- * <code>.vem</code> the vector metadata, <code>.veq</code> the quantized vector data, <code>.vex</code> the
- * hnsw graph data, <code>.veb</code>the binary quantized vector data, and
- * <code>.vemb</code>the binary quantized vector metadata</td>
+ * <code>.vem</code> the vector metadata, <code>.veq</code> the quantized vector data, and <code>.vex</code> the
+ * hnsw graph data.</td>
* </tr>
* </table>
*
@@ -419,7 +418,7 @@
* need skipping, especially conjunctions.
* <li>In version 10.1, block encoding changed to be optimized for int[] storage instead of
* long[].
- * <li>In version 10.2, new vector asymmetric binary quantization that for HNSW and flat formats.
+ * <li>In version 10.3, the index of block tree changed to be specialized trie instead of FST.
* </ul>
*
* <a id="Limitations"></a>
@@ -434,4 +433,4 @@
* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
* VInt} values which have no limit. </div>
*/
-package org.apache.lucene.codecs.lucene102;
+package org.apache.lucene.codecs.lucene103;
diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
index b9b7b2dbd41..f725de389a6 100644
--- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
+++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java
@@ -215,8 +215,8 @@ public abstract class VectorizationProvider {
Set.of(
"org.apache.lucene.codecs.hnsw.FlatVectorScorerUtil",
"org.apache.lucene.util.VectorUtil",
- "org.apache.lucene.codecs.lucene101.Lucene101PostingsReader",
- "org.apache.lucene.codecs.lucene101.PostingIndexInput",
+ "org.apache.lucene.codecs.lucene103.Lucene103PostingsReader",
+ "org.apache.lucene.codecs.lucene103.PostingIndexInput",
"org.apache.lucene.tests.util.TestSysoutsLimits");
private static void ensureCaller() {
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index b1d283cc389..e478c08e05e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -21,8 +21,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -399,10 +399,10 @@ public class PhraseQuery extends Query {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene101PostingsReader.BlockPostingsEnum#nextPosition()}.
+ * Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene101PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene103PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
@@ -410,7 +410,7 @@ public class PhraseQuery extends Query {
private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
/**
- * Number of simple operations in {@link Lucene101PostingsReader.BlockPostingsEnum#nextPosition()}
+ * Number of simple operations in {@link Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}
* when no seek or buffer refill is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
index de304ea2722..a08b36413a5 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java
@@ -18,7 +18,7 @@ package org.apache.lucene.search;
import java.util.stream.LongStream;
import java.util.stream.StreamSupport;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.FixedBitSet;
@@ -28,7 +28,7 @@ import org.apache.lucene.util.PriorityQueue;
class ScorerUtil {
private static final Class<?> DEFAULT_IMPACTS_ENUM_CLASS =
- Lucene101PostingsFormat.getImpactsEnumImpl();
+ Lucene103PostingsFormat.getImpactsEnumImpl();
private static final Class<?> DEFAULT_ACCEPT_DOCS_CLASS =
new FixedBitSet(1).asReadOnlyBits().getClass();
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index f7a246c7684..b85efc3c37f 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene101.Lucene101Codec
+org.apache.lucene.codecs.lucene103.Lucene103Codec
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 874ebafd971..12d44d82f7d 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat
+org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForDeltaUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForDeltaUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java
index d41ab472ea6..224731bdaf1 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForDeltaUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForDeltaUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
@@ -68,7 +68,7 @@ public class TestForDeltaUtil extends LuceneTestCase {
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
PostingDecodingUtil pdu =
- Lucene101PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
ForDeltaUtil forDeltaUtil = new ForDeltaUtil();
for (int i = 0; i < iterations; ++i) {
int base = 0;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java
index d93cb0ab3ec..935eff6e8fd 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestForUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
@@ -71,7 +71,7 @@ public class TestForUtil extends LuceneTestCase {
// decode
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
PostingDecodingUtil pdu =
- Lucene101PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
ForUtil forUtil = new ForUtil();
for (int i = 0; i < iterations; ++i) {
final int bitsPerValue = in.readByte();
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java
similarity index 88%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java
index 6b0ff1fe5e3..210fde9fa23 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestLucene101PostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestLucene103PostingsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import java.util.Arrays;
@@ -22,9 +22,9 @@ import java.util.Collections;
import java.util.List;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader.MutableImpactList;
-import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Stats;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader.MutableImpactList;
+import org.apache.lucene.codecs.lucene103.blocktree.FieldReader;
+import org.apache.lucene.codecs.lucene103.blocktree.Stats;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -41,11 +41,11 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
-public class TestLucene101PostingsFormat extends BasePostingsFormatTestCase {
+public class TestLucene103PostingsFormat extends BasePostingsFormatTestCase {
@Override
protected Codec getCodec() {
- return TestUtil.alwaysPostingsFormat(new Lucene101PostingsFormat());
+ return TestUtil.alwaysPostingsFormat(new Lucene103PostingsFormat());
}
public void testVInt15() throws IOException {
@@ -54,9 +54,9 @@ public class TestLucene101PostingsFormat extends BasePostingsFormatTestCase {
ByteArrayDataInput in = new ByteArrayDataInput();
for (int i : new int[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE}) {
out.reset(bytes);
- Lucene101PostingsWriter.writeVInt15(out, i);
+ Lucene103PostingsWriter.writeVInt15(out, i);
in.reset(bytes, 0, out.getPosition());
- assertEquals(i, Lucene101PostingsReader.readVInt15(in));
+ assertEquals(i, Lucene103PostingsReader.readVInt15(in));
assertEquals(out.getPosition(), in.getPosition());
}
}
@@ -67,9 +67,9 @@ public class TestLucene101PostingsFormat extends BasePostingsFormatTestCase {
ByteArrayDataInput in = new ByteArrayDataInput();
for (long i : new long[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE, Long.MAX_VALUE}) {
out.reset(bytes);
- Lucene101PostingsWriter.writeVLong15(out, i);
+ Lucene103PostingsWriter.writeVLong15(out, i);
in.reset(bytes, 0, out.getPosition());
- assertEquals(i, Lucene101PostingsReader.readVLong15(in));
+ assertEquals(i, Lucene103PostingsReader.readVLong15(in));
assertEquals(out.getPosition(), in.getPosition());
}
}
@@ -141,13 +141,13 @@ public class TestLucene101PostingsFormat extends BasePostingsFormatTestCase {
}
try (Directory dir = newDirectory()) {
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
- Lucene101PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);
+ Lucene103PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);
}
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
byte[] b = new byte[Math.toIntExact(in.length())];
in.readBytes(b, 0, b.length);
List<Impact> impacts2 =
- Lucene101PostingsReader.readImpacts(
+ Lucene103PostingsReader.readImpacts(
new ByteArrayDataInput(b),
new MutableImpactList(impacts.size() + random().nextInt(3)));
assertEquals(impacts, impacts2);
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPForUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPForUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java
index 4e9ab4b55ee..61879034ffe 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPForUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
@@ -41,7 +41,7 @@ public class TestPForUtil extends LuceneTestCase {
IndexInput in = d.openInput("test.bin", IOContext.READONCE);
PostingDecodingUtil pdu =
- Lucene101PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
+ Lucene103PostingsReader.VECTORIZATION_PROVIDER.newPostingDecodingUtil(in);
final PForUtil pforUtil = new PForUtil();
for (int i = 0; i < iterations; ++i) {
if (random().nextInt(5) == 0) {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPostingsUtil.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPostingsUtil.java
rename to lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java
index 5d02d0561e3..32edd3608ba 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene101/TestPostingsUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/TestPostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene101;
+package org.apache.lucene.codecs.lucene103;
import java.io.IOException;
import org.apache.lucene.store.Directory;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene103/blocktree/TestTrie.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/blocktree/TestTrie.java
new file mode 100644
index 00000000000..2d6fb53ea64
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene103/blocktree/TestTrie.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene103.blocktree;
+
+import static org.apache.lucene.codecs.lucene103.blocktree.TrieBuilder.ChildSaveStrategy;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.function.Supplier;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.tests.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
+import org.junit.Assert;
+
+public class TestTrie extends LuceneTestCase {
+
+ public void testStrategyChoose() {
+ // bits use 32 bytes while reverse_array use 31 bytes, choose reverse_array
+ assertSame(ChildSaveStrategy.REVERSE_ARRAY, ChildSaveStrategy.choose(0, 255, 226));
+ // bits use 32 bytes while array use 31 bytes, choose array
+ assertSame(ChildSaveStrategy.ARRAY, ChildSaveStrategy.choose(0, 255, 32));
+ // array and bits both use 32 position bytes, we choose bits.
+ assertSame(ChildSaveStrategy.BITS, ChildSaveStrategy.choose(0, 255, 33));
+ // reverse_array and bits both use 32 position bytes, we choose bits.
+ assertSame(ChildSaveStrategy.BITS, ChildSaveStrategy.choose(0, 255, 225));
+ }
+
+ public void testRandomTerms() throws Exception {
+ Supplier<byte[]> supplier = TestTrie::randomBytes;
+ testTrieBuilder(supplier, atLeast(1000));
+ testTrieLookup(supplier, 12);
+ }
+
+ public void testVeryLongTerms() throws Exception {
+ Supplier<byte[]> supplier =
+ () -> {
+ byte[] bytes = new byte[65535];
+ for (int i = 1; i < bytes.length; i++) {
+ bytes[i] = (byte) random().nextInt(i / 256 + 1);
+ }
+ return bytes;
+ };
+ testTrieLookup(supplier, 5);
+ }
+
+ public void testOneByteTerms() throws Exception {
+ // heavily test single byte terms to generate various label distribution.
+ Supplier<byte[]> supplier = () -> new byte[] {(byte) random().nextInt()};
+ int round = atLeast(50);
+ for (int i = 0; i < round; i++) {
+ testTrieLookup(supplier, 10);
+ }
+ }
+
+ private void testTrieBuilder(Supplier<byte[]> randomBytesSupplier, int count) {
+ Map<BytesRef, TrieBuilder.Output> expected = new TreeMap<>();
+ expected.put(new BytesRef(""), new TrieBuilder.Output(0L, false, new BytesRef("emptyOutput")));
+ for (int i = 0; i < count; i++) {
+ BytesRef key = new BytesRef(randomBytesSupplier.get());
+ TrieBuilder.Output value =
+ new TrieBuilder.Output(
+ random().nextLong(1L << 62),
+ random().nextBoolean(),
+ new BytesRef(randomBytesSupplier.get()));
+ expected.put(key, value);
+ }
+
+ TrieBuilder trieBuilder =
+ TrieBuilder.bytesRefToTrie(
+ new BytesRef(""), new TrieBuilder.Output(0L, false, new BytesRef("emptyOutput")));
+ for (var entry : expected.entrySet()) {
+ if (entry.getKey().equals(new BytesRef(""))) {
+ continue;
+ }
+ TrieBuilder add = TrieBuilder.bytesRefToTrie(entry.getKey(), entry.getValue());
+ trieBuilder.append(add);
+ Assert.assertThrows(IllegalStateException.class, () -> add.append(trieBuilder));
+ Assert.assertThrows(IllegalStateException.class, () -> trieBuilder.append(add));
+ }
+ Map<BytesRef, TrieBuilder.Output> actual = new TreeMap<>();
+ trieBuilder.visit(actual::put);
+ assertEquals(expected, actual);
+ }
+
+ private void testTrieLookup(Supplier<byte[]> randomBytesSupplier, int round) throws IOException {
+ for (int iter = 1; iter <= round; iter++) {
+ Map<BytesRef, TrieBuilder.Output> expected = new TreeMap<>();
+ expected.put(
+ new BytesRef(""), new TrieBuilder.Output(0L, false, new BytesRef("emptyOutput")));
+ int n = 1 << iter;
+ for (int i = 0; i < n; i++) {
+ BytesRef key = new BytesRef(randomBytesSupplier.get());
+ TrieBuilder.Output value =
+ new TrieBuilder.Output(
+ random().nextLong(1L << 62),
+ random().nextBoolean(),
+ random().nextBoolean() ? null : new BytesRef(randomBytesSupplier.get()));
+ expected.put(key, value);
+ }
+
+ TrieBuilder trieBuilder =
+ TrieBuilder.bytesRefToTrie(
+ new BytesRef(""), new TrieBuilder.Output(0L, false, new BytesRef("emptyOutput")));
+ for (var entry : expected.entrySet()) {
+ if (entry.getKey().equals(new BytesRef(""))) {
+ continue;
+ }
+ TrieBuilder add = TrieBuilder.bytesRefToTrie(entry.getKey(), entry.getValue());
+ trieBuilder.append(add);
+ Assert.assertThrows(IllegalStateException.class, () -> add.append(trieBuilder));
+ Assert.assertThrows(IllegalStateException.class, () -> trieBuilder.append(add));
+ }
+
+ try (Directory directory = newDirectory()) {
+ try (IndexOutput index = directory.createOutput("index", IOContext.DEFAULT);
+ IndexOutput meta = directory.createOutput("meta", IOContext.DEFAULT)) {
+ trieBuilder.save(meta, index);
+ assertThrows(IllegalStateException.class, () -> trieBuilder.save(meta, index));
+ assertThrows(
+ IllegalStateException.class,
+ () ->
+ trieBuilder.append(
+ TrieBuilder.bytesRefToTrie(
+ new BytesRef(), new TrieBuilder.Output(0L, true, null))));
+ }
+
+ try (IndexInput indexIn = directory.openInput("index", IOContext.DEFAULT);
+ IndexInput metaIn = directory.openInput("meta", IOContext.DEFAULT)) {
+ long start = metaIn.readVLong();
+ long rootFP = metaIn.readVLong();
+ long end = metaIn.readVLong();
+ TrieReader reader = new TrieReader(indexIn.slice("outputs", start, end - start), rootFP);
+
+ for (Map.Entry<BytesRef, TrieBuilder.Output> entry : expected.entrySet()) {
+ assertResult(reader, entry.getKey(), entry.getValue());
+ }
+
+ int testNotFound = atLeast(100);
+ for (int i = 0; i < testNotFound; i++) {
+ BytesRef key = new BytesRef(randomBytes());
+ while (expected.containsKey(key)) {
+ key = new BytesRef(randomBytes());
+ }
+ BytesRef lastK = new BytesRef();
+ for (BytesRef k : expected.keySet()) {
+ if (k.compareTo(key) > 0) {
+ assert lastK.compareTo(key) < 0;
+ int mismatch1 =
+ Arrays.mismatch(
+ lastK.bytes,
+ lastK.offset,
+ lastK.offset + lastK.length,
+ key.bytes,
+ key.offset,
+ key.offset + key.length);
+ int mismatch2 =
+ Arrays.mismatch(
+ k.bytes,
+ k.offset,
+ k.offset + k.length,
+ key.bytes,
+ key.offset,
+ key.offset + key.length);
+ assertNotFoundOnLevelN(reader, key, Math.max(mismatch1, mismatch2));
+ break;
+ }
+ lastK = k;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ private static byte[] randomBytes() {
+ byte[] bytes = new byte[random().nextInt(256) + 1];
+ for (int i = 1; i < bytes.length; i++) {
+ bytes[i] = (byte) random().nextInt(1 << (i % 9));
+ }
+ return bytes;
+ }
+
+ private static void assertResult(TrieReader reader, BytesRef term, TrieBuilder.Output expected)
+ throws IOException {
+ TrieReader.Node parent = reader.root;
+ TrieReader.Node child = new TrieReader.Node();
+ for (int i = 0; i < term.length; i++) {
+ TrieReader.Node found = reader.lookupChild(term.bytes[i + term.offset] & 0xFF, parent, child);
+ Assert.assertNotNull(found);
+ parent = child;
+ child = new TrieReader.Node();
+ }
+ assertTrue(parent.hasOutput());
+ assertEquals(expected.fp(), parent.outputFp);
+ assertEquals(expected.hasTerms(), parent.hasTerms);
+ if (expected.floorData() == null) {
+ assertFalse(parent.isFloor());
+ } else {
+ byte[] bytes = new byte[expected.floorData().length];
+ parent.floorData(reader).readBytes(bytes, 0, bytes.length);
+ assertArrayEquals(BytesRef.deepCopyOf(expected.floorData()).bytes, bytes);
+ }
+ }
+
+ private static void assertNotFoundOnLevelN(TrieReader reader, BytesRef term, int n)
+ throws IOException {
+ TrieReader.Node parent = reader.root;
+ TrieReader.Node child = new TrieReader.Node();
+ for (int i = 0; i < term.length; i++) {
+ TrieReader.Node found = reader.lookupChild(term.bytes[i + term.offset] & 0xFF, parent, child);
+ if (i == n) {
+ assertNull(found);
+ break;
+ }
+ Assert.assertNotNull(found);
+ parent = child;
+ child = new TrieReader.Node();
+ }
+ }
+}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
index 69fbf96f6da..b41988631e6 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
@@ -18,7 +18,7 @@ package org.apache.lucene.codecs.lucene90;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DirectoryReader;
@@ -31,7 +31,7 @@ import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene101Codec(Lucene101Codec.Mode.BEST_COMPRESSION);
+ return new Lucene103Codec(Lucene103Codec.Mode.BEST_COMPRESSION);
}
/**
@@ -42,7 +42,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
for (int i = 0; i < 10; i++) {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setCodec(
- new Lucene101Codec(RandomPicks.randomFrom(random(), Lucene101Codec.Mode.values())));
+ new Lucene103Codec(RandomPicks.randomFrom(random(), Lucene103Codec.Mode.values())));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(new StoredField("field1", "value1"));
@@ -72,7 +72,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
expectThrows(
NullPointerException.class,
() -> {
- new Lucene101Codec(null);
+ new Lucene103Codec(null);
});
expectThrows(
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java b/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
index d06330c2926..c6d9d88c7e4 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestForTooMuchCloning.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.index;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
@@ -35,17 +36,25 @@ public class TestForTooMuchCloning extends LuceneTestCase {
// during merging and searching:
public void test() throws Exception {
final MockDirectoryWrapper dir = newMockDirectory();
+ dir.setVerboseClone(false); // set true to view clone stacks.
final TieredMergePolicy tmp = new TieredMergePolicy();
tmp.setMaxMergeAtOnce(2);
+ AtomicInteger segmentsMerged = new AtomicInteger();
final RandomIndexWriter w =
new RandomIndexWriter(
random(),
dir,
newIndexWriterConfig(new MockAnalyzer(random()))
+ // to reduce flakiness on merge clone count
+ .setMergeScheduler(new SerialMergeScheduler())
.setMaxBufferedDocs(2)
- // use a FilterMP otherwise RIW will randomly reconfigure
- // the MP while the test runs
- .setMergePolicy(new FilterMergePolicy(tmp)));
+ .setMergePolicy(
+ new OneMergeWrappingMergePolicy(
+ tmp,
+ oneMerge -> {
+ segmentsMerged.addAndGet(oneMerge.segments.size());
+ return oneMerge;
+ })));
final int numDocs = 20;
for (int docs = 0; docs < numDocs; docs++) {
StringBuilder sb = new StringBuilder();
@@ -62,7 +71,7 @@ public class TestForTooMuchCloning extends LuceneTestCase {
// System.out.println("merge clone count=" + cloneCount);
assertTrue(
"too many calls to IndexInput.clone during merging: " + dir.getInputCloneCount(),
- dir.getInputCloneCount() < 500);
+ dir.getInputCloneCount() < (r.leaves().size() + segmentsMerged.get()) * 50);
final IndexSearcher s = newSearcher(r);
// important: set this after newSearcher, it might have run checkindex
@@ -85,7 +94,7 @@ public class TestForTooMuchCloning extends LuceneTestCase {
// perform.
assertTrue(
"too many calls to IndexInput.clone during TermRangeQuery: " + queryCloneCount,
- queryCloneCount <= Math.max(s.getLeafContexts().size(), s.getSlices().length) * 5);
+ queryCloneCount <= Math.max(s.getLeafContexts().size(), s.getSlices().length) * 7);
r.close();
dir.close();
}
diff --git a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java
index 6c914dfcc03..0a9fb4b1ce9 100644
--- a/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java
+++ b/lucene/core/src/test/org/apache/lucene/internal/vectorization/TestPostingDecodingUtil.java
@@ -16,7 +16,7 @@
*/
package org.apache.lucene.internal.vectorization;
-import org.apache.lucene.codecs.lucene101.ForUtil;
+import org.apache.lucene.codecs.lucene103.ForUtil;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
index c1360a9d9a7..e075aaccc51 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@@ -261,10 +261,10 @@ class TermIntervalsSource extends IntervalsSource {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene101PostingsReader.BlockPostingsEnum#nextPosition()}.
+ * Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene101PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene103PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
@@ -272,7 +272,7 @@ class TermIntervalsSource extends IntervalsSource {
private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
/**
- * Number of simple operations in {@link Lucene101PostingsReader.BlockPostingsEnum#nextPosition()}
+ * Number of simple operations in {@link Lucene103PostingsReader.BlockPostingsEnum#nextPosition()}
* when no seek or buffer refill is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java
index d4c08667cbb..32072af9283 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java
@@ -22,7 +22,7 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.LiveFieldValues;
@@ -66,15 +66,15 @@ public class IDVersionPostingsFormat extends PostingsFormat {
public IDVersionPostingsFormat() {
this(
- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
+ Lucene103BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene103BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
}
public IDVersionPostingsFormat(int minTermsInBlock, int maxTermsInBlock) {
super("IDVersion");
this.minTermsInBlock = minTermsInBlock;
this.maxTermsInBlock = maxTermsInBlock;
- Lucene90BlockTreeTermsWriter.validateSettings(minTermsInBlock, maxTermsInBlock);
+ Lucene103BlockTreeTermsWriter.validateSettings(minTermsInBlock, maxTermsInBlock);
}
@Override
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java
index 6d498ba950a..c416e0083c5 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/VersionBlockTreeTermsWriter.java
@@ -24,7 +24,7 @@ import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.NormsProducer;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.Fields;
@@ -83,9 +83,9 @@ import org.apache.lucene.util.fst.Util;
*/
/**
- * This is just like {@link Lucene90BlockTreeTermsWriter}, except it also stores a version per term,
- * and adds a method to its TermsEnum implementation to seekExact only if the version is >= the
- * specified version. The version is added to the terms index to avoid seeking if no term in the
+ * This is just like {@link Lucene103BlockTreeTermsWriter}, except it also stores a version per
+ * term, and adds a method to its TermsEnum implementation to seekExact only if the version is >=
+ * the specified version. The version is added to the terms index to avoid seeking if no term in the
* block has a high enough version. The term blocks file is .tiv and the terms index extension is
* .tipv.
*
@@ -181,7 +181,7 @@ public final class VersionBlockTreeTermsWriter extends FieldsConsumer {
int minItemsInBlock,
int maxItemsInBlock)
throws IOException {
- Lucene90BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
+ Lucene103BlockTreeTermsWriter.validateSettings(minItemsInBlock, maxItemsInBlock);
maxDoc = state.segmentInfo.maxDoc();
final String termsFileName =
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
index e9dde1705b9..5e419016945 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion101PostingsFormat.java
@@ -20,7 +20,7 @@ import org.apache.lucene.codecs.PostingsFormat;
/**
* {@link CompletionPostingsFormat} for {@link
- * org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat}
+ * org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat}
*
* @lucene.experimental
*/
@@ -32,6 +32,6 @@ public class Completion101PostingsFormat extends CompletionPostingsFormat {
@Override
protected PostingsFormat delegatePostingsFormat() {
- return PostingsFormat.forName("Lucene101");
+ return PostingsFormat.forName("Lucene103");
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
index 7c5a0bcf254..17ceff0d4c2 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
@@ -28,9 +28,9 @@ import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;
import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -38,7 +38,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene101PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
+ * Customized version of {@link Lucene103PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
*/
public final class LuceneFixedGap extends PostingsFormat {
final int termIndexInterval;
@@ -54,7 +54,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene101PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene103PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -91,7 +91,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene101PostingsReader(state);
+ PostingsReaderBase postings = new Lucene103PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
index ff5e0fc07dd..741ce732518 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene101PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene103PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval, but forcing high docfreq terms to be indexed
* terms.
*/
@@ -59,7 +59,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene101PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene103PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -100,7 +100,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene101PostingsReader(state);
+ PostingsReaderBase postings = new Lucene103PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
index 7899f3d5436..93dd141b061 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene101PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene103PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval.
*/
public final class LuceneVarGapFixedInterval extends PostingsFormat {
@@ -56,7 +56,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene101PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene103PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -95,7 +95,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene101PostingsReader(state);
+ PostingsReaderBase postings = new Lucene103PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
index 8770e0d81fa..e660ccc4478 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
@@ -35,10 +35,10 @@ import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
-import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsReader;
+import org.apache.lucene.codecs.lucene103.blocktree.Lucene103BlockTreeTermsWriter;
import org.apache.lucene.codecs.memory.FSTTermsReader;
import org.apache.lucene.codecs.memory.FSTTermsWriter;
import org.apache.lucene.index.FieldInfo;
@@ -121,7 +121,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
random.nextInt(); // consume a random for buffersize
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(state);
final FieldsConsumer fields;
final int t1 = random.nextInt(4);
@@ -151,7 +151,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
boolean success = false;
try {
fields =
- new Lucene90BlockTreeTermsWriter(
+ new Lucene103BlockTreeTermsWriter(
state, postingsWriter, minTermsInBlock, maxTermsInBlock);
success = true;
} finally {
@@ -289,7 +289,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);
}
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene103PostingsReader(state);
final FieldsProducer fields;
final int t1 = random.nextInt(4);
@@ -311,7 +311,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
boolean success = false;
try {
- fields = new Lucene90BlockTreeTermsReader(postingsReader, state);
+ fields = new Lucene103BlockTreeTermsReader(postingsReader, state);
success = true;
} finally {
if (!success) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
index 2b1613dc8e4..63f7bcbf2cb 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsReader;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsWriter;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsReader;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsWriter;
import org.apache.lucene.codecs.uniformsplit.BlockDecoder;
import org.apache.lucene.codecs.uniformsplit.BlockEncoder;
import org.apache.lucene.codecs.uniformsplit.IndexDictionary;
@@ -68,7 +68,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState segmentWriteState) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene101PostingsWriter(segmentWriteState);
+ PostingsWriterBase postingsWriter = new Lucene103PostingsWriter(segmentWriteState);
boolean success = false;
try {
FieldsConsumer fieldsConsumer = createFieldsConsumer(segmentWriteState, postingsWriter);
@@ -146,7 +146,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState segmentReadState) throws IOException {
- PostingsReaderBase postingsReader = new Lucene101PostingsReader(segmentReadState);
+ PostingsReaderBase postingsReader = new Lucene103PostingsReader(segmentReadState);
boolean success = false;
try {
FieldsProducer fieldsProducer = createFieldsProducer(segmentReadState, postingsReader);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
index 1ace55a98a0..13db0d34e78 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
@@ -38,7 +38,7 @@ import java.util.TimeZone;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
+import org.apache.lucene.codecs.lucene103.Lucene103Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
@@ -190,7 +190,7 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule {
codec = CompressingCodec.randomInstance(random);
} else if ("Lucene100".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene100"))) {
- codec = new Lucene101Codec(RandomPicks.randomFrom(random, Lucene101Codec.Mode.values()));
+ codec = new Lucene103Codec(RandomPicks.randomFrom(random, Lucene103Codec.Mode.values()));
} else if (!"random".equals(TEST_CODEC)) {
codec = Codec.forName(TEST_CODEC);
} else if ("random".equals(TEST_POSTINGSFORMAT)) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
index e557314b59d..b59f5bff342 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
@@ -62,8 +62,8 @@ import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
-import org.apache.lucene.codecs.lucene101.Lucene101Codec;
-import org.apache.lucene.codecs.lucene101.Lucene101PostingsFormat;
+import org.apache.lucene.codecs.lucene103.Lucene103Codec;
+import org.apache.lucene.codecs.lucene103.Lucene103PostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
@@ -1341,7 +1341,7 @@ public final class TestUtil {
* different from {@link Codec#getDefault()} because that is randomized.
*/
public static Codec getDefaultCodec() {
- return new Lucene101Codec();
+ return new Lucene103Codec();
}
/**
@@ -1349,7 +1349,7 @@ public final class TestUtil {
* Lucene.
*/
public static PostingsFormat getDefaultPostingsFormat() {
- return new Lucene101PostingsFormat();
+ return new Lucene103PostingsFormat();
}
/**
@@ -1360,7 +1360,7 @@ public final class TestUtil {
*/
public static PostingsFormat getDefaultPostingsFormat(
int minItemsPerBlock, int maxItemsPerBlock) {
- return new Lucene101PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
+ return new Lucene103PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
}
/** Returns a random postings format that supports term ordinals */