You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2024/07/31 16:12:25 UTC
(lucene) branch branch_9x updated (539274f4530 -> 59b7c28eb47)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a change to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
from 539274f4530 Introduce IndexSearcher#searchLeaf(LeafReaderContext, Weight, Collector) method (#13603)
new 24de36cc28c Bump the window size of disjunction from 2,048 to 4,096. (#13605)
new 59b7c28eb47 Inline skip data into postings lists (#13585)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
gradle/generation/forUtil.gradle | 27 +-
lucene/CHANGES.txt | 6 +
.../src/generated/checksums/generateForUtil99.json | 4 +
lucene/backward-codecs/src/java/module-info.java | 7 +-
.../backward_codecs}/lucene99/ForDeltaUtil.java | 2 +-
.../lucene/backward_codecs}/lucene99/ForUtil.java | 2 +-
.../backward_codecs}/lucene99/Lucene99Codec.java | 29 +-
.../lucene99/Lucene99PostingsFormat.java | 38 +-
.../lucene99/Lucene99PostingsReader.java | 20 +-
.../lucene99/Lucene99ScoreSkipReader.java | 2 +-
.../lucene99/Lucene99SkipReader.java | 11 +-
.../lucene99/Lucene99SkipWriter.java | 5 +-
.../lucene/backward_codecs}/lucene99/PForUtil.java | 11 +-
.../backward_codecs}/lucene99/PostingsUtil.java | 2 +-
.../backward_codecs}/lucene99/gen_ForUtil.py | 2 +-
.../backward_codecs}/lucene99/package-info.java | 30 +-
.../services/org.apache.lucene.codecs.Codec | 1 +
.../org.apache.lucene.codecs.PostingsFormat | 1 +
.../lucene90/TestLucene90PostingsFormat.java | 19 +-
.../lucene99/Lucene99PostingsWriter.java | 18 +-
.../lucene99/Lucene99RWPostingsFormat.java} | 58 +-
.../lucene99/TestForDeltaUtil.java | 2 +-
.../backward_codecs}/lucene99/TestForUtil.java | 2 +-
...stLucene99HnswScalarQuantizedVectorsFormat.java | 1 -
.../lucene99/TestLucene99PostingsFormat.java | 16 +-
.../backward_codecs}/lucene99/TestPForUtil.java | 2 +-
.../lucene99/TestPostingsUtil.java | 2 +-
.../TestInt8HnswBackwardsCompatibility.java | 2 +-
.../blocktreeords/BlockTreeOrdsPostingsFormat.java | 10 +-
.../lucene/codecs/memory/DirectPostingsFormat.java | 8 +-
.../lucene/codecs/memory/FSTPostingsFormat.java | 8 +-
.../uniformsplit/DeltaBaseTermStateSerializer.java | 21 +-
.../uniformsplit/UniformSplitPostingsFormat.java | 8 +-
.../lucene/codecs/uniformsplit/package-info.java | 2 +-
.../bitvectors/TestHnswBitVectorsFormat.java | 4 +-
.../lucene90/tests/MockTermStateFactory.java | 2 +-
.../src/generated/checksums/generateForUtil.json | 4 +-
lucene/core/src/java/module-info.java | 7 +-
.../src/java/org/apache/lucene/codecs/Codec.java | 2 +-
.../codecs/CompetitiveImpactAccumulator.java | 6 +-
.../lucene90/Lucene90StoredFieldsFormat.java | 4 +-
.../{lucene99 => lucene912}/ForDeltaUtil.java | 7 +-
.../codecs/{lucene99 => lucene912}/ForUtil.java | 2 +-
.../Lucene912Codec.java} | 50 +-
.../Lucene912PostingsFormat.java} | 131 +-
.../codecs/lucene912/Lucene912PostingsReader.java | 2083 ++++++++++++++++++++
.../Lucene912PostingsWriter.java} | 389 ++--
.../codecs/{lucene99 => lucene912}/PForUtil.java | 2 +-
.../{lucene99 => lucene912}/PostingsUtil.java | 6 +-
.../codecs/{lucene99 => lucene912}/gen_ForUtil.py | 2 +-
.../{lucene99 => lucene912}/package-info.java | 23 +-
.../lucene/codecs/lucene99/package-info.java | 411 +---
.../org/apache/lucene/search/BooleanScorer.java | 2 +-
.../apache/lucene/search/MaxScoreBulkScorer.java | 2 +-
.../java/org/apache/lucene/search/PhraseQuery.java | 10 +-
.../services/org.apache.lucene.codecs.Codec | 2 +-
.../org.apache.lucene.codecs.PostingsFormat | 2 +-
...tLucene90StoredFieldsFormatHighCompression.java | 8 +-
.../lucene912/TestLucene912PostingsFormat.java} | 63 +-
.../TestLucene99HnswQuantizedVectorsFormat.java | 5 +-
.../TestLucene99ScalarQuantizedVectorScorer.java | 3 +-
.../TestLucene99ScalarQuantizedVectorsFormat.java | 3 +-
.../apache/lucene/index/TestLazyProxSkipping.java | 195 --
.../lucene/search/TestMaxScoreBulkScorer.java | 26 +-
.../queries/intervals/TermIntervalsSource.java | 10 +-
lucene/suggest/src/java/module-info.java | 3 +-
...ormat.java => Completion912PostingsFormat.java} | 16 +-
.../document/Completion99PostingsFormat.java | 6 +-
.../org.apache.lucene.codecs.PostingsFormat | 1 +
.../search/suggest/document/TestSuggestField.java | 2 +-
.../tests/codecs/blockterms/LuceneFixedGap.java | 12 +-
.../blockterms/LuceneVarGapDocFreqInterval.java | 12 +-
.../blockterms/LuceneVarGapFixedInterval.java | 12 +-
.../mockrandom/MockRandomPostingsFormat.java | 8 +-
.../UniformSplitRot13PostingsFormat.java | 8 +-
.../util/TestRuleSetupAndRestoreClassEnv.java | 8 +-
.../org/apache/lucene/tests/util/TestUtil.java | 10 +-
77 files changed, 2798 insertions(+), 1145 deletions(-)
create mode 100644 lucene/backward-codecs/src/generated/checksums/generateForUtil99.json
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/ForDeltaUtil.java (98%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/ForUtil.java (99%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/Lucene99Codec.java (82%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/Lucene99PostingsFormat.java (95%)
rename lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/Lucene99PostingsReader.java (98%)
rename lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/Lucene99ScoreSkipReader.java (98%)
rename lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/Lucene99SkipReader.java (92%)
rename lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/Lucene99SkipWriter.java (97%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/PForUtil.java (94%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/PostingsUtil.java (97%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/gen_ForUtil.py (99%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/java/org/apache/lucene/backward_codecs}/lucene99/package-info.java (93%)
copy lucene/{core/src/java/org/apache/lucene/codecs => backward-codecs/src/test/org/apache/lucene/backward_codecs}/lucene99/Lucene99PostingsWriter.java (95%)
copy lucene/{codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java => backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java} (53%)
rename lucene/{core/src/test/org/apache/lucene/codecs => backward-codecs/src/test/org/apache/lucene/backward_codecs}/lucene99/TestForDeltaUtil.java (98%)
rename lucene/{core/src/test/org/apache/lucene/codecs => backward-codecs/src/test/org/apache/lucene/backward_codecs}/lucene99/TestForUtil.java (98%)
rename lucene/{core/src/test/org/apache/lucene/codecs => backward-codecs/src/test/org/apache/lucene/backward_codecs}/lucene99/TestLucene99PostingsFormat.java (90%)
rename lucene/{core/src/test/org/apache/lucene/codecs => backward-codecs/src/test/org/apache/lucene/backward_codecs}/lucene99/TestPForUtil.java (98%)
rename lucene/{core/src/test/org/apache/lucene/codecs => backward-codecs/src/test/org/apache/lucene/backward_codecs}/lucene99/TestPostingsUtil.java (97%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99 => lucene912}/ForDeltaUtil.java (92%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99 => lucene912}/ForUtil.java (99%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99/Lucene99Codec.java => lucene912/Lucene912Codec.java} (75%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99/Lucene99PostingsFormat.java => lucene912/Lucene912PostingsFormat.java} (81%)
create mode 100644 lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsReader.java
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99/Lucene99PostingsWriter.java => lucene912/Lucene912PostingsWriter.java} (55%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99 => lucene912}/PForUtil.java (99%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99 => lucene912}/PostingsUtil.java (92%)
rename lucene/core/src/java/org/apache/lucene/codecs/{lucene99 => lucene912}/gen_ForUtil.py (99%)
copy lucene/core/src/java/org/apache/lucene/codecs/{lucene99 => lucene912}/package-info.java (95%)
copy lucene/{backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java => core/src/test/org/apache/lucene/codecs/lucene912/TestLucene912PostingsFormat.java} (73%)
delete mode 100644 lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
copy lucene/suggest/src/java/org/apache/lucene/search/suggest/document/{Completion99PostingsFormat.java => Completion912PostingsFormat.java} (69%)
(lucene) 02/02: Inline skip data into postings lists (#13585)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 59b7c28eb474bc223b7ede1e386651136468f3cc
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Wed Jul 31 17:18:28 2024 +0200
Inline skip data into postings lists (#13585)
This updates the postings format in order to inline skip data into postings. This format is generally similar to the current `Lucene99PostingsFormat`, e.g. it shares the same block encoding logic, but it has a few differences:
- Skip data is inlined into postings to make the access pattern more sequential.
- There are only 2 levels of skip data: on every block (128 docs) and every 32 blocks (4,096 docs).
In general, I found that the fact that skip data is inlined may slow down a bit queries that don't need skip data at all (e.g. `CountOrXXX` tasks that never advance of consult impacts) and speed up a bit queries that advance by small intervals. The fact that the greatest level only allows skipping 4096 docs at once means that we're slower at advancing by large intervals, but data suggests that it doesn't significantly hurt performance.
---
gradle/generation/forUtil.gradle | 27 +-
lucene/CHANGES.txt | 6 +
.../src/generated/checksums/generateForUtil99.json | 4 +
lucene/backward-codecs/src/java/module-info.java | 7 +-
.../backward_codecs}/lucene99/ForDeltaUtil.java | 2 +-
.../lucene/backward_codecs}/lucene99/ForUtil.java | 2 +-
.../backward_codecs}/lucene99/Lucene99Codec.java | 29 +-
.../lucene99/Lucene99PostingsFormat.java | 38 +-
.../lucene99/Lucene99PostingsReader.java | 20 +-
.../lucene99/Lucene99ScoreSkipReader.java | 2 +-
.../lucene99/Lucene99SkipReader.java | 11 +-
.../lucene99/Lucene99SkipWriter.java | 5 +-
.../lucene/backward_codecs}/lucene99/PForUtil.java | 11 +-
.../backward_codecs}/lucene99/PostingsUtil.java | 2 +-
.../backward_codecs}/lucene99/gen_ForUtil.py | 2 +-
.../backward_codecs}/lucene99/package-info.java | 30 +-
.../services/org.apache.lucene.codecs.Codec | 1 +
.../org.apache.lucene.codecs.PostingsFormat | 1 +
.../lucene90/TestLucene90PostingsFormat.java | 19 +-
.../lucene99/Lucene99PostingsWriter.java | 18 +-
.../lucene99/Lucene99RWPostingsFormat.java} | 58 +-
.../lucene99/TestForDeltaUtil.java | 2 +-
.../backward_codecs}/lucene99/TestForUtil.java | 2 +-
...stLucene99HnswScalarQuantizedVectorsFormat.java | 1 -
.../lucene99/TestLucene99PostingsFormat.java | 16 +-
.../backward_codecs}/lucene99/TestPForUtil.java | 2 +-
.../lucene99/TestPostingsUtil.java | 2 +-
.../TestInt8HnswBackwardsCompatibility.java | 2 +-
.../blocktreeords/BlockTreeOrdsPostingsFormat.java | 10 +-
.../lucene/codecs/memory/DirectPostingsFormat.java | 8 +-
.../lucene/codecs/memory/FSTPostingsFormat.java | 8 +-
.../uniformsplit/DeltaBaseTermStateSerializer.java | 21 +-
.../uniformsplit/UniformSplitPostingsFormat.java | 8 +-
.../lucene/codecs/uniformsplit/package-info.java | 2 +-
.../bitvectors/TestHnswBitVectorsFormat.java | 4 +-
.../lucene90/tests/MockTermStateFactory.java | 2 +-
.../src/generated/checksums/generateForUtil.json | 4 +-
lucene/core/src/java/module-info.java | 7 +-
.../src/java/org/apache/lucene/codecs/Codec.java | 2 +-
.../codecs/CompetitiveImpactAccumulator.java | 6 +-
.../lucene90/Lucene90StoredFieldsFormat.java | 4 +-
.../{lucene99 => lucene912}/ForDeltaUtil.java | 7 +-
.../codecs/{lucene99 => lucene912}/ForUtil.java | 2 +-
.../Lucene912Codec.java} | 50 +-
.../Lucene912PostingsFormat.java} | 131 +-
.../codecs/lucene912/Lucene912PostingsReader.java | 2083 ++++++++++++++++++++
.../Lucene912PostingsWriter.java} | 389 ++--
.../codecs/{lucene99 => lucene912}/PForUtil.java | 2 +-
.../{lucene99 => lucene912}/PostingsUtil.java | 6 +-
.../codecs/{lucene99 => lucene912}/gen_ForUtil.py | 2 +-
.../{lucene99 => lucene912}/package-info.java | 23 +-
.../lucene/codecs/lucene99/package-info.java | 411 +---
.../java/org/apache/lucene/search/PhraseQuery.java | 10 +-
.../services/org.apache.lucene.codecs.Codec | 2 +-
.../org.apache.lucene.codecs.PostingsFormat | 2 +-
...tLucene90StoredFieldsFormatHighCompression.java | 8 +-
.../lucene912/TestLucene912PostingsFormat.java} | 63 +-
.../TestLucene99HnswQuantizedVectorsFormat.java | 5 +-
.../TestLucene99ScalarQuantizedVectorScorer.java | 3 +-
.../TestLucene99ScalarQuantizedVectorsFormat.java | 3 +-
.../apache/lucene/index/TestLazyProxSkipping.java | 195 --
.../queries/intervals/TermIntervalsSource.java | 10 +-
lucene/suggest/src/java/module-info.java | 3 +-
...ormat.java => Completion912PostingsFormat.java} | 16 +-
.../document/Completion99PostingsFormat.java | 6 +-
.../org.apache.lucene.codecs.PostingsFormat | 1 +
.../search/suggest/document/TestSuggestField.java | 2 +-
.../tests/codecs/blockterms/LuceneFixedGap.java | 12 +-
.../blockterms/LuceneVarGapDocFreqInterval.java | 12 +-
.../blockterms/LuceneVarGapFixedInterval.java | 12 +-
.../mockrandom/MockRandomPostingsFormat.java | 8 +-
.../UniformSplitRot13PostingsFormat.java | 8 +-
.../util/TestRuleSetupAndRestoreClassEnv.java | 8 +-
.../org/apache/lucene/tests/util/TestUtil.java | 10 +-
74 files changed, 2783 insertions(+), 1130 deletions(-)
diff --git a/gradle/generation/forUtil.gradle b/gradle/generation/forUtil.gradle
index e1428234875..5de1d850a02 100644
--- a/gradle/generation/forUtil.gradle
+++ b/gradle/generation/forUtil.gradle
@@ -23,7 +23,7 @@ configure(project(":lucene:core")) {
description "Regenerate gen_ForUtil.py"
group "generation"
- def genDir = file("src/java/org/apache/lucene/codecs/lucene99")
+ def genDir = file("src/java/org/apache/lucene/codecs/lucene912")
def genScript = file("${genDir}/gen_ForUtil.py")
def genOutput = file("${genDir}/ForUtil.java")
@@ -96,5 +96,30 @@ configure(project(":lucene:backward-codecs")) {
andThenTasks: ["spotlessJava", "spotlessJavaApply"],
mustRunBefore: [ "compileJava" ]
])
+
+ task generateForUtil99Internal() {
+ description "Regenerate gen_ForUtil.py"
+ group "generation"
+
+ def genDir = file("src/java/org/apache/lucene/backward_codecs/lucene99")
+ def genScript = file("${genDir}/gen_ForUtil.py")
+ def genOutput = file("${genDir}/ForUtil.java")
+
+ inputs.file genScript
+ outputs.file genOutput
+
+ doLast {
+ quietExec {
+ workingDir genDir
+ executable project.externalTool("python3")
+ args = [ '-B', genScript ]
+ }
+ }
+ }
+
+ regenerate.dependsOn wrapWithPersistentChecksums(generateForUtil99Internal, [
+ andThenTasks: ["spotlessJava", "spotlessJavaApply"],
+ mustRunBefore: [ "compileJava" ]
+ ])
}
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6d17ba1d048..bb08a573249 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -65,6 +65,12 @@ Optimizations
* GITHUB#13582: Stop requiring MaxScoreBulkScorer's outer window from having at
least INNER_WINDOW_SIZE docs. (Adrien Grand)
+* GITHUB#13585: Lucene912PostingsFormat, the new default postings format, now
+ only has 2 levels of skip data, which are inlined into postings instead of
+ being stored at the end of postings lists. This translates into better
+ performance for queries that need skipping such as conjunctions.
+ (Adrien Grand)
+
Changes in runtime behavior
---------------------
diff --git a/lucene/backward-codecs/src/generated/checksums/generateForUtil99.json b/lucene/backward-codecs/src/generated/checksums/generateForUtil99.json
new file mode 100644
index 00000000000..872e9c7bd6d
--- /dev/null
+++ b/lucene/backward-codecs/src/generated/checksums/generateForUtil99.json
@@ -0,0 +1,4 @@
+{
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForUtil.java": "f31797842f047626df6a1a6b97167bec60269fec",
+ "lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/gen_ForUtil.py": "325f2610974b0e76e278b6445405a098a3763feb"
+}
\ No newline at end of file
diff --git a/lucene/backward-codecs/src/java/module-info.java b/lucene/backward-codecs/src/java/module-info.java
index 51a9128e38c..b5daf785bed 100644
--- a/lucene/backward-codecs/src/java/module-info.java
+++ b/lucene/backward-codecs/src/java/module-info.java
@@ -35,6 +35,7 @@ module org.apache.lucene.backward_codecs {
exports org.apache.lucene.backward_codecs.lucene92;
exports org.apache.lucene.backward_codecs.lucene94;
exports org.apache.lucene.backward_codecs.lucene95;
+ exports org.apache.lucene.backward_codecs.lucene99;
exports org.apache.lucene.backward_codecs.packed;
exports org.apache.lucene.backward_codecs.store;
@@ -44,7 +45,8 @@ module org.apache.lucene.backward_codecs {
provides org.apache.lucene.codecs.PostingsFormat with
org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat,
org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat,
- org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat;
+ org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat,
+ org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
org.apache.lucene.backward_codecs.lucene90.Lucene90HnswVectorsFormat,
org.apache.lucene.backward_codecs.lucene91.Lucene91HnswVectorsFormat,
@@ -61,5 +63,6 @@ module org.apache.lucene.backward_codecs {
org.apache.lucene.backward_codecs.lucene91.Lucene91Codec,
org.apache.lucene.backward_codecs.lucene92.Lucene92Codec,
org.apache.lucene.backward_codecs.lucene94.Lucene94Codec,
- org.apache.lucene.backward_codecs.lucene95.Lucene95Codec;
+ org.apache.lucene.backward_codecs.lucene95.Lucene95Codec,
+ org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForDeltaUtil.java
similarity index 98%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForDeltaUtil.java
index 2403c997435..2fa67ebf40e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForDeltaUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForUtil.java
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForUtil.java
index 3126cdb02c0..cc089751784 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/ForUtil.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99Codec.java
similarity index 82%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99Codec.java
index ce6c40f9402..d540abc85c2 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99Codec.java
@@ -14,12 +14,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.util.Objects;
-import org.apache.lucene.codecs.*;
-import org.apache.lucene.codecs.lucene90.*;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
@@ -98,7 +119,7 @@ public class Lucene99Codec extends Codec {
super("Lucene99");
this.storedFieldsFormat =
new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);
- this.defaultPostingsFormat = new Lucene99PostingsFormat();
+ this.defaultPostingsFormat = new Lucene912PostingsFormat();
this.defaultDVFormat = new Lucene90DocValuesFormat();
this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java
similarity index 95%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java
index 877746641b4..7ff614d684e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsFormat.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
@@ -24,7 +24,6 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.MultiLevelSkipListWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.IndexOptions;
@@ -339,7 +338,7 @@ import org.apache.lucene.util.packed.PackedInts;
*
* @lucene.experimental
*/
-public final class Lucene99PostingsFormat extends PostingsFormat {
+public class Lucene99PostingsFormat extends PostingsFormat {
/**
* Filename extension for document number, frequencies, and skip data. See chapter: <a
@@ -374,28 +373,9 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
- private final int minTermBlockSize;
- private final int maxTermBlockSize;
-
/** Creates {@code Lucene99PostingsFormat} with default settings. */
public Lucene99PostingsFormat() {
- this(
- Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
- Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
- }
-
- /**
- * Creates {@code Lucene99PostingsFormat} with custom values for {@code minBlockSize} and {@code
- * maxBlockSize} passed to block terms dictionary.
- *
- * @see
- * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
- */
- public Lucene99PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
super("Lucene99");
- Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
- this.minTermBlockSize = minTermBlockSize;
- this.maxTermBlockSize = maxTermBlockSize;
}
@Override
@@ -405,19 +385,7 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
- boolean success = false;
- try {
- FieldsConsumer ret =
- new Lucene90BlockTreeTermsWriter(
- state, postingsWriter, minTermBlockSize, maxTermBlockSize);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsWriter);
- }
- }
+ throw new UnsupportedOperationException();
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsReader.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsReader.java
index 05a4825ba4b..9537707354f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsReader.java
@@ -14,23 +14,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
-import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_START;
+import static org.apache.lucene.backward_codecs.lucene99.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.VERSION_START;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.PostingsReaderBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Impacts;
import org.apache.lucene.index.ImpactsEnum;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScoreSkipReader.java
similarity index 98%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScoreSkipReader.java
index f2882c085c4..4c655d545fd 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99ScoreSkipReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99ScoreSkipReader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.util.AbstractList;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99SkipReader.java
similarity index 92%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99SkipReader.java
index a6cb7c3a811..567fe655374 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipReader.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99SkipReader.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.util.Arrays;
@@ -61,6 +61,7 @@ public class Lucene99SkipReader extends MultiLevelSkipListReader {
private long lastDocPointer;
private int lastPosBufferUpto;
+ /** Sole constructor. */
public Lucene99SkipReader(
IndexInput skipStream,
int maxSkipLevels,
@@ -98,6 +99,7 @@ public class Lucene99SkipReader extends MultiLevelSkipListReader {
return df % ForUtil.BLOCK_SIZE == 0 ? df - 1 : df;
}
+ /** Initialize state. */
public void init(
long skipPointer, long docBasePointer, long posBasePointer, long payBasePointer, int df)
throws IOException {
@@ -125,22 +127,27 @@ public class Lucene99SkipReader extends MultiLevelSkipListReader {
return lastDocPointer;
}
+ /** Returns the pointer in the pos file. */
public long getPosPointer() {
return lastPosPointer;
}
+ /** Return the start offset in the position block. */
public int getPosBufferUpto() {
return lastPosBufferUpto;
}
+ /** Returns the pointer in the pay file. */
public long getPayPointer() {
return lastPayPointer;
}
+ /** Return the number of bytes in the pay block that belongs to docs from the previous block. */
public int getPayloadByteUpto() {
return lastPayloadByteUpto;
}
+ /** Return the next skip doc, no skipping can be performed until this doc. */
public int getNextSkipDoc() {
return skipDoc[0];
}
@@ -199,7 +206,7 @@ public class Lucene99SkipReader extends MultiLevelSkipListReader {
return delta;
}
- // The default impl skips impacts
+ /** Read impacts. The default implementation skips them. */
protected void readImpacts(int level, IndexInput skipStream) throws IOException {
skipStream.skipBytes(skipStream.readVInt());
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99SkipWriter.java
similarity index 97%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java
rename to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99SkipWriter.java
index 93efda8b0da..495cf07d9af 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99SkipWriter.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/Lucene99SkipWriter.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.util.Arrays;
@@ -66,6 +66,7 @@ public final class Lucene99SkipWriter extends MultiLevelSkipListWriter {
private boolean fieldHasOffsets;
private boolean fieldHasPayloads;
+ /** Sole constructor. */
public Lucene99SkipWriter(
int maxSkipLevels,
int blockSize,
@@ -92,6 +93,7 @@ public final class Lucene99SkipWriter extends MultiLevelSkipListWriter {
}
}
+ /** Reset state for the given index options. */
public void setField(
boolean fieldHasPositions, boolean fieldHasOffsets, boolean fieldHasPayloads) {
this.fieldHasPositions = fieldHasPositions;
@@ -211,6 +213,7 @@ public final class Lucene99SkipWriter extends MultiLevelSkipListWriter {
competitiveFreqNorms.clear();
}
+ /** Write impacts to the given output. */
public static void writeImpacts(CompetitiveImpactAccumulator acc, DataOutput out)
throws IOException {
Collection<Impact> impacts = acc.getCompetitiveFreqNormPairs();
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PForUtil.java
similarity index 94%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PForUtil.java
index 93ebb3d0f9d..9eddd3a0c55 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import java.util.Arrays;
@@ -28,15 +28,6 @@ final class PForUtil {
private static final int MAX_EXCEPTIONS = 7;
- // IDENTITY_PLUS_ONE[i] == i + 1
- private static final long[] IDENTITY_PLUS_ONE = new long[ForUtil.BLOCK_SIZE];
-
- static {
- for (int i = 0; i < ForUtil.BLOCK_SIZE; ++i) {
- IDENTITY_PLUS_ONE[i] = i + 1;
- }
- }
-
static boolean allEqual(long[] l) {
for (int i = 1; i < ForUtil.BLOCK_SIZE; ++i) {
if (l[i] != l[0]) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PostingsUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java
similarity index 97%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/PostingsUtil.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java
index 678754047b6..7b95bada5bc 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PostingsUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.store.IndexInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/gen_ForUtil.py
similarity index 99%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/gen_ForUtil.py
index cd81d493570..97a3a49a4ff 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/gen_ForUtil.py
@@ -40,7 +40,7 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/package-info.java
similarity index 93%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
copy to lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/package-info.java
index 2df2cfc9a30..6c1b0b44a07 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/package-info.java
@@ -151,17 +151,17 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
+ * <li>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Term dictionary}.
+ * A dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
+ * <li>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Term Frequency
+ * data}. For each term in the dictionary, the numbers of all the documents that contain that
+ * term, and the frequency of the term in that document, unless frequencies are omitted
+ * ({@link org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
+ * <li>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Term Proximity
+ * data}. For each term in the dictionary, the positions that the term occurs in each
+ * document. Note that this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
* each field in each document, a value is stored that is multiplied into the score for hits
* on that field.
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
@@ -425,4 +425,4 @@
* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
* VInt} values which have no limit. </div>
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 76ebb5de33f..7c20015a704 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -23,3 +23,4 @@ org.apache.lucene.backward_codecs.lucene91.Lucene91Codec
org.apache.lucene.backward_codecs.lucene92.Lucene92Codec
org.apache.lucene.backward_codecs.lucene94.Lucene94Codec
org.apache.lucene.backward_codecs.lucene95.Lucene95Codec
+org.apache.lucene.backward_codecs.lucene99.Lucene99Codec
diff --git a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 18ac359d2ae..9733362abe7 100644
--- a/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/backward-codecs/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -16,3 +16,4 @@
org.apache.lucene.backward_codecs.lucene50.Lucene50PostingsFormat
org.apache.lucene.backward_codecs.lucene84.Lucene84PostingsFormat
org.apache.lucene.backward_codecs.lucene90.Lucene90PostingsFormat
+org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
index 7965bc9c778..814285a8895 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
@@ -23,12 +23,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import org.apache.lucene.backward_codecs.lucene90.Lucene90ScoreSkipReader.MutableImpactList;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99SkipWriter;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
import org.apache.lucene.codecs.lucene90.blocktree.Stats;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99SkipWriter;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -77,22 +76,6 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
d.close();
}
- private void shouldFail(int minItemsInBlock, int maxItemsInBlock) {
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- new Lucene99PostingsFormat(minItemsInBlock, maxItemsInBlock);
- });
- }
-
- public void testInvalidBlockSizes() throws Exception {
- shouldFail(0, 0);
- shouldFail(10, 8);
- shouldFail(-1, 10);
- shouldFail(10, -1);
- shouldFail(10, 12);
- }
-
public void testImpactSerialization() throws IOException {
// omit norms and omit freqs
doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsWriter.java
similarity index 95%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsWriter.java
index 61949cfe227..44ff826e159 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99PostingsWriter.java
@@ -14,22 +14,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
-import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.backward_codecs.lucene99.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexFileNames;
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java
similarity index 53%
copy from lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
copy to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java
index 0a18b75ac00..f513562cb35 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/Lucene99RWPostingsFormat.java
@@ -14,38 +14,49 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.memory;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.codecs.FieldsConsumer;
-import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
-import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
-/** FST term dict + Lucene50PBF */
-public final class FSTPostingsFormat extends PostingsFormat {
- public FSTPostingsFormat() {
- super("FST50");
+public class Lucene99RWPostingsFormat extends Lucene99PostingsFormat {
+
+ private final int minTermBlockSize;
+ private final int maxTermBlockSize;
+
+ /** Creates {@code Lucene99PostingsFormat} with default settings. */
+ public Lucene99RWPostingsFormat() {
+ this(
+ Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
+ Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
}
- @Override
- public String toString() {
- return getName();
+ /**
+ * Creates {@code Lucene99PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * maxBlockSize} passed to block terms dictionary.
+ *
+ * @see
+ * Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
+ */
+ public Lucene99RWPostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ super();
+ Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
+ this.minTermBlockSize = minTermBlockSize;
+ this.maxTermBlockSize = maxTermBlockSize;
}
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
-
boolean success = false;
try {
- FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
+ FieldsConsumer ret =
+ new Lucene90BlockTreeTermsWriter(
+ state, postingsWriter, minTermBlockSize, maxTermBlockSize);
success = true;
return ret;
} finally {
@@ -54,19 +65,4 @@ public final class FSTPostingsFormat extends PostingsFormat {
}
}
}
-
- @Override
- public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
- boolean success = false;
- try {
- FieldsProducer ret = new FSTTermsReader(state, postingsReader);
- success = true;
- return ret;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(postingsReader);
- }
- }
- }
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestForDeltaUtil.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestForDeltaUtil.java
index 29448ff1e41..e21eaab461c 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForDeltaUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestForDeltaUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestForUtil.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestForUtil.java
index 2f179c743af..85e0dcaebc4 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestForUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
index 0e0d63963cc..01d5b4ab17f 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99HnswScalarQuantizedVectorsFormat.java
@@ -19,7 +19,6 @@ package org.apache.lucene.backward_codecs.lucene99;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.apache.lucene.tests.index.BaseKnnVectorsFormatTestCase;
public class TestLucene99HnswScalarQuantizedVectorsFormat extends BaseKnnVectorsFormatTestCase {
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java
similarity index 90%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java
index 341805e8a3e..e954af88c3b 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestLucene99PostingsFormat.java
@@ -14,22 +14,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
-import static org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.readImpacts;
+import static org.apache.lucene.backward_codecs.lucene99.Lucene99ScoreSkipReader.readImpacts;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
import org.apache.lucene.codecs.lucene90.blocktree.Stats;
-import org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.index.*;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.Impact;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -41,7 +45,7 @@ import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
public class TestLucene99PostingsFormat extends BasePostingsFormatTestCase {
- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene99PostingsFormat());
+ private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene99RWPostingsFormat());
@Override
protected Codec getCodec() {
@@ -77,7 +81,7 @@ public class TestLucene99PostingsFormat extends BasePostingsFormatTestCase {
expectThrows(
IllegalArgumentException.class,
() -> {
- new Lucene99PostingsFormat(minItemsInBlock, maxItemsInBlock);
+ new Lucene99RWPostingsFormat(minItemsInBlock, maxItemsInBlock);
});
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestPForUtil.java
similarity index 98%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestPForUtil.java
index 16f719e93b2..528d5d2b960 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPForUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestPForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
import java.io.IOException;
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPostingsUtil.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestPostingsUtil.java
similarity index 97%
rename from lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPostingsUtil.java
rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestPostingsUtil.java
index 3a69d35fc5c..eb5c132fc08 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestPostingsUtil.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene99/TestPostingsUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.store.Directory;
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java
index d840a00ca5b..8db406df992 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestInt8HnswBackwardsCompatibility.java
@@ -20,9 +20,9 @@ import static org.apache.lucene.backward_index.TestBasicBackwardsCompatibility.a
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import java.io.IOException;
+import org.apache.lucene.backward_codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
import org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
import org.apache.lucene.document.Document;
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
index d389372d8f8..db704148573 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/blocktreeords/BlockTreeOrdsPostingsFormat.java
@@ -23,13 +23,13 @@ import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
-/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene99PostingsWriter}. */
+/** Uses {@link OrdsBlockTreeTermsWriter} with {@link Lucene912PostingsWriter}. */
public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
private final int minTermBlockSize;
@@ -67,7 +67,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(state);
boolean success = false;
try {
@@ -84,7 +84,7 @@ public class BlockTreeOrdsPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene912PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new OrdsBlockTreeTermsReader(postingsReader, state);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
index 7b0e53d35b5..ed740908a06 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
@@ -24,7 +24,7 @@ import java.util.TreeMap;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
import org.apache.lucene.index.BaseTermsEnum;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.Fields;
@@ -53,7 +53,7 @@ import org.apache.lucene.util.automaton.Transition;
// - or: longer dense skip lists than just next byte?
/**
- * Wraps {@link Lucene99PostingsFormat} format for on-disk storage, but then at read time loads and
+ * Wraps {@link Lucene912PostingsFormat} format for on-disk storage, but then at read time loads and
* stores all terms and postings directly in RAM as byte[], int[].
*
* <p><b>WARNING</b>: This is exceptionally RAM intensive: it makes no effort to compress the
@@ -96,12 +96,12 @@ public final class DirectPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- return PostingsFormat.forName("Lucene99").fieldsConsumer(state);
+ return PostingsFormat.forName("Lucene912").fieldsConsumer(state);
}
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- FieldsProducer postings = PostingsFormat.forName("Lucene99").fieldsProducer(state);
+ FieldsProducer postings = PostingsFormat.forName("Lucene912").fieldsProducer(state);
if (state.context.context != IOContext.Context.MERGE) {
FieldsProducer loadedPostings;
try {
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
index 0a18b75ac00..b184f85b176 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTPostingsFormat.java
@@ -22,8 +22,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -41,7 +41,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(state);
boolean success = false;
try {
@@ -57,7 +57,7 @@ public final class FSTPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene912PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new FSTTermsReader(state, postingsReader);
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
index ae01216ccf6..1f876e5e9d1 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/DeltaBaseTermStateSerializer.java
@@ -17,13 +17,13 @@
package org.apache.lucene.codecs.uniformsplit;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.BLOCK_SIZE;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.TermState;
@@ -34,7 +34,7 @@ import org.apache.lucene.util.RamUsageEstimator;
/**
* {@link TermState} serializer which encodes each file pointer as a delta relative to a base file
- * pointer. It differs from {@link Lucene99PostingsWriter#encodeTerm} which encodes each file
+ * pointer. It differs from {@link Lucene912PostingsWriter#encodeTerm} which encodes each file
* pointer as a delta relative to the previous file pointer.
*
* <p>It automatically sets the base file pointer to the first valid file pointer for doc start FP,
@@ -95,7 +95,7 @@ public class DeltaBaseTermStateSerializer implements Accountable {
/**
* Writes a {@link BlockTermState} to the provided {@link DataOutput}.
*
- * <p>Simpler variant of {@link Lucene99PostingsWriter#encodeTerm(DataOutput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene912PostingsWriter#encodeTerm(DataOutput, FieldInfo,
* BlockTermState, boolean)}.
*/
public void writeTermState(
@@ -140,15 +140,12 @@ public class DeltaBaseTermStateSerializer implements Accountable {
termStatesOutput.writeVLong(intTermState.lastPosBlockOffset);
}
}
- if (intTermState.skipOffset != -1) {
- termStatesOutput.writeVLong(intTermState.skipOffset);
- }
}
/**
* Reads a {@link BlockTermState} from the provided {@link DataInput}.
*
- * <p>Simpler variant of {@link Lucene99PostingsReader#decodeTerm(DataInput, FieldInfo,
+ * <p>Simpler variant of {@link Lucene912PostingsReader#decodeTerm(DataInput, FieldInfo,
* BlockTermState, boolean)}.
*
* @param reuse {@link BlockTermState} to reuse; or null to create a new one.
@@ -190,9 +187,6 @@ public class DeltaBaseTermStateSerializer implements Accountable {
intTermState.lastPosBlockOffset = termStatesInput.readVLong();
}
}
- if (intTermState.docFreq > BLOCK_SIZE) {
- intTermState.skipOffset = termStatesInput.readVLong();
- }
return intTermState;
}
@@ -210,7 +204,6 @@ public class DeltaBaseTermStateSerializer implements Accountable {
termState.docStartFP = 0;
termState.posStartFP = 0;
termState.payStartFP = 0;
- termState.skipOffset = -1;
termState.lastPosBlockOffset = -1;
termState.singletonDocID = -1;
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
index 8e2b7bb7d54..c8a19bf9da9 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/UniformSplitPostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.util.IOUtils;
@@ -113,7 +113,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(state);
boolean success = false;
try {
FieldsConsumer termsWriter =
@@ -130,7 +130,7 @@ public class UniformSplitPostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene912PostingsReader(state);
boolean success = false;
try {
FieldsProducer termsReader =
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
index a0fa67508f3..dc77bc710a1 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/uniformsplit/package-info.java
@@ -28,7 +28,7 @@
* org.apache.lucene.search.PhraseQuery})
* <li>Quite efficient for {@link org.apache.lucene.search.PrefixQuery}
* <li>Not efficient for spell-check and {@link org.apache.lucene.search.FuzzyQuery}, in this case
- * prefer {@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}
+ * prefer {@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat}
* </ul>
*/
package org.apache.lucene.codecs.uniformsplit;
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/bitvectors/TestHnswBitVectorsFormat.java b/lucene/codecs/src/test/org/apache/lucene/codecs/bitvectors/TestHnswBitVectorsFormat.java
index 8ce34b0e566..ab20ee67c8c 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/bitvectors/TestHnswBitVectorsFormat.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/bitvectors/TestHnswBitVectorsFormat.java
@@ -22,7 +22,7 @@ import java.io.IOException;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.KnnByteVectorField;
@@ -42,7 +42,7 @@ import org.apache.lucene.tests.index.BaseIndexFileFormatTestCase;
public class TestHnswBitVectorsFormat extends BaseIndexFileFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene99Codec() {
+ return new Lucene912Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new HnswBitVectorsFormat();
diff --git a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
index dde59993f6e..51891b9d0ad 100644
--- a/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
+++ b/lucene/codecs/src/test/org/apache/lucene/codecs/lucene90/tests/MockTermStateFactory.java
@@ -17,7 +17,7 @@
package org.apache.lucene.codecs.lucene90.tests;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.IntBlockTermState;
/** Test utility class to create mock {@link IntBlockTermState}. */
public class MockTermStateFactory {
diff --git a/lucene/core/src/generated/checksums/generateForUtil.json b/lucene/core/src/generated/checksums/generateForUtil.json
index 527070d7bf0..752285f4d7f 100644
--- a/lucene/core/src/generated/checksums/generateForUtil.json
+++ b/lucene/core/src/generated/checksums/generateForUtil.json
@@ -1,4 +1,4 @@
{
- "lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java": "1292ad354d255b1272ffd3db684aa2ddb2bc49ec",
- "lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py": "ab7b63a1b73986cc04e43de1c8f474b97aef5116"
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForUtil.java": "5ff856e80cab30f9e5704aa89f3197f017d07624",
+ "lucene/core/src/java/org/apache/lucene/codecs/lucene912/gen_ForUtil.py": "3ccf92b3ddbff6340a13e8a55090bfb900dc7be2"
}
\ No newline at end of file
diff --git a/lucene/core/src/java/module-info.java b/lucene/core/src/java/module-info.java
index 245cead327c..e017ba17727 100644
--- a/lucene/core/src/java/module-info.java
+++ b/lucene/core/src/java/module-info.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
/** Lucene Core. */
@SuppressWarnings("module") // the test framework is compiled after the core...
@@ -34,6 +34,7 @@ module org.apache.lucene.core {
exports org.apache.lucene.codecs.lucene94;
exports org.apache.lucene.codecs.lucene95;
exports org.apache.lucene.codecs.lucene99;
+ exports org.apache.lucene.codecs.lucene912;
exports org.apache.lucene.codecs.perfield;
exports org.apache.lucene.codecs;
exports org.apache.lucene.document;
@@ -72,7 +73,7 @@ module org.apache.lucene.core {
provides org.apache.lucene.analysis.TokenizerFactory with
org.apache.lucene.analysis.standard.StandardTokenizerFactory;
provides org.apache.lucene.codecs.Codec with
- Lucene99Codec;
+ Lucene912Codec;
provides org.apache.lucene.codecs.DocValuesFormat with
org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
provides org.apache.lucene.codecs.KnnVectorsFormat with
@@ -80,7 +81,7 @@ module org.apache.lucene.core {
org.apache.lucene.codecs.lucene99.Lucene99HnswScalarQuantizedVectorsFormat,
org.apache.lucene.codecs.lucene99.Lucene99ScalarQuantizedVectorsFormat;
provides org.apache.lucene.codecs.PostingsFormat with
- org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
+ org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
provides org.apache.lucene.index.SortFieldProvider with
org.apache.lucene.search.SortField.Provider,
org.apache.lucene.search.SortedNumericSortField.Provider,
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
index 0b654a134a6..e5a5dac8ff5 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/Codec.java
@@ -55,7 +55,7 @@ public abstract class Codec implements NamedSPILoader.NamedSPI {
return LOADER;
}
- static Codec defaultCodec = LOADER.lookup("Lucene99");
+ static Codec defaultCodec = LOADER.lookup("Lucene912");
}
private final String name;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java
index 37b9fc8bd20..77d38d290f8 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/CompetitiveImpactAccumulator.java
@@ -18,8 +18,6 @@ package org.apache.lucene.codecs;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -106,7 +104,7 @@ public final class CompetitiveImpactAccumulator {
}
/** Get the set of competitive freq and norm pairs, ordered by increasing freq and norm. */
- public Collection<Impact> getCompetitiveFreqNormPairs() {
+ public List<Impact> getCompetitiveFreqNormPairs() {
List<Impact> impacts = new ArrayList<>();
int maxFreqForLowerNorms = 0;
for (int i = 0; i < maxFreqs.length; ++i) {
@@ -126,7 +124,7 @@ public final class CompetitiveImpactAccumulator {
for (Impact impact : impacts) {
add(impact, freqNormPairs);
}
- return Collections.unmodifiableSet(freqNormPairs);
+ return List.copyOf(freqNormPairs);
}
private void add(Impact newEntry, TreeSet<Impact> freqNormPairs) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90StoredFieldsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90StoredFieldsFormat.java
index 5646724546a..ce0310d6396 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90StoredFieldsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90StoredFieldsFormat.java
@@ -49,9 +49,9 @@ import org.apache.lucene.util.packed.DirectMonotonicWriter;
*
* <pre class="prettyprint">
* // the default: for high performance
- * indexWriterConfig.setCodec(new Lucene99Codec(Mode.BEST_SPEED));
+ * indexWriterConfig.setCodec(new Lucene912Codec(Mode.BEST_SPEED));
* // instead for higher performance (but slower):
- * // indexWriterConfig.setCodec(new Lucene99Codec(Mode.BEST_COMPRESSION));
+ * // indexWriterConfig.setCodec(new Lucene912Codec(Mode.BEST_COMPRESSION));
* </pre>
*
* <p><b>File formats</b>
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForDeltaUtil.java
similarity index 92%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForDeltaUtil.java
index 2403c997435..8b9aedcfb2b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForDeltaUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForDeltaUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
@@ -75,4 +75,9 @@ public class ForDeltaUtil {
forUtil.decodeAndPrefixSum(bitsPerValue, in, base, longs);
}
}
+
+ void skip(DataInput in) throws IOException {
+ final int bitsPerValue = Byte.toUnsignedInt(in.readByte());
+ in.skipBytes(forUtil.numBytes(bitsPerValue));
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForUtil.java
index 3126cdb02c0..63ee7baaf10 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/ForUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/ForUtil.java
@@ -16,7 +16,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912Codec.java
similarity index 75%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912Codec.java
index ce6c40f9402..cb4ef755a6b 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99Codec.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912Codec.java
@@ -14,25 +14,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.util.Objects;
-import org.apache.lucene.codecs.*;
-import org.apache.lucene.codecs.lucene90.*;
+import org.apache.lucene.codecs.Codec;
+import org.apache.lucene.codecs.CompoundFormat;
+import org.apache.lucene.codecs.DocValuesFormat;
+import org.apache.lucene.codecs.FieldInfosFormat;
+import org.apache.lucene.codecs.FilterCodec;
+import org.apache.lucene.codecs.KnnVectorsFormat;
+import org.apache.lucene.codecs.LiveDocsFormat;
+import org.apache.lucene.codecs.NormsFormat;
+import org.apache.lucene.codecs.PointsFormat;
+import org.apache.lucene.codecs.PostingsFormat;
+import org.apache.lucene.codecs.SegmentInfoFormat;
+import org.apache.lucene.codecs.StoredFieldsFormat;
+import org.apache.lucene.codecs.TermVectorsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90NormsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90PointsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
+import org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
+import org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
/**
- * Implements the Lucene 9.9 index format
+ * Implements the Lucene 9.12 index format
*
* <p>If you want to reuse functionality of this codec in another codec, extend {@link FilterCodec}.
*
* @see org.apache.lucene.codecs.lucene99 package documentation for file format details.
* @lucene.experimental
*/
-public class Lucene99Codec extends Codec {
+public class Lucene912Codec extends Codec {
/** Configuration option for the codec. */
public enum Mode {
@@ -60,7 +80,7 @@ public class Lucene99Codec extends Codec {
new PerFieldPostingsFormat() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
- return Lucene99Codec.this.getPostingsFormatForField(field);
+ return Lucene912Codec.this.getPostingsFormatForField(field);
}
};
@@ -69,7 +89,7 @@ public class Lucene99Codec extends Codec {
new PerFieldDocValuesFormat() {
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
- return Lucene99Codec.this.getDocValuesFormatForField(field);
+ return Lucene912Codec.this.getDocValuesFormatForField(field);
}
};
@@ -78,14 +98,14 @@ public class Lucene99Codec extends Codec {
new PerFieldKnnVectorsFormat() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
- return Lucene99Codec.this.getKnnVectorsFormatForField(field);
+ return Lucene912Codec.this.getKnnVectorsFormatForField(field);
}
};
private final StoredFieldsFormat storedFieldsFormat;
/** Instantiates a new codec. */
- public Lucene99Codec() {
+ public Lucene912Codec() {
this(Mode.BEST_SPEED);
}
@@ -94,11 +114,11 @@ public class Lucene99Codec extends Codec {
*
* @param mode stored fields compression mode to use for newly flushed/merged segments.
*/
- public Lucene99Codec(Mode mode) {
- super("Lucene99");
+ public Lucene912Codec(Mode mode) {
+ super("Lucene912");
this.storedFieldsFormat =
new Lucene90StoredFieldsFormat(Objects.requireNonNull(mode).storedMode);
- this.defaultPostingsFormat = new Lucene99PostingsFormat();
+ this.defaultPostingsFormat = new Lucene912PostingsFormat();
this.defaultDVFormat = new Lucene90DocValuesFormat();
this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat();
}
@@ -151,7 +171,7 @@ public class Lucene99Codec extends Codec {
/**
* Returns the postings format that should be used for writing new segments of <code>field</code>.
*
- * <p>The default implementation always returns "Lucene99".
+ * <p>The default implementation always returns "Lucene912".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation,
@@ -164,7 +184,7 @@ public class Lucene99Codec extends Codec {
* Returns the docvalues format that should be used for writing new segments of <code>field</code>
* .
*
- * <p>The default implementation always returns "Lucene99".
+ * <p>The default implementation always returns "Lucene90".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
@@ -176,7 +196,7 @@ public class Lucene99Codec extends Codec {
/**
* Returns the vectors format that should be used for writing new segments of <code>field</code>
*
- * <p>The default implementation always returns "Lucene95".
+ * <p>The default implementation always returns "Lucene99HnswVectorsFormat".
*
* <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility:
* future version of Lucene are only guaranteed to be able to read the default implementation.
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsFormat.java
similarity index 81%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsFormat.java
index 877746641b4..1c452175b07 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsFormat.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsFormat.java
@@ -14,14 +14,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
-import org.apache.lucene.codecs.MultiLevelSkipListWriter;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
@@ -36,7 +35,7 @@ import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
/**
- * Lucene 9.9 postings format, which encodes postings in packed integer blocks for fast decode.
+ * Lucene 9.12 postings format, which encodes postings in packed integer blocks for fast decode.
*
* <p>Basic idea:
*
@@ -48,7 +47,7 @@ import org.apache.lucene.util.packed.PackedInts;
* <p>In VInt blocks, integers are encoded as {@link DataOutput#writeVInt VInt}: the block
* size is variable.
* <li><b>Block structure</b>:
- * <p>When the postings are long enough, Lucene99PostingsFormat will try to encode most
+ * <p>When the postings are long enough, Lucene912PostingsFormat will try to encode most
* integer data as a packed block.
* <p>Take a term with 259 documents as an example, the first 256 document ids are encoded as
* two packed blocks, while the remaining 3 are encoded as one VInt block.
@@ -57,10 +56,10 @@ import org.apache.lucene.util.packed.PackedInts;
* <p>This strategy is applied to pairs: <document number, frequency>, <position,
* payload length>, <position, offset start, offset length>, and <position,
* payload length, offsetstart, offset length>.
- * <li><b>Skipdata settings</b>:
- * <p>The structure of skip table is quite similar to previous version of Lucene. Skip
- * interval is the same as block size, and each skip entry points to the beginning of each
- * block. However, for the first block, skip data is omitted.
+ * <li><b>Skipdata</b>:
+ * <p>Skipdata is interleaved with blocks on 2 levels. Level 0 skip data is interleaved
+ * between every packed block. Level 1 skip data is interleaved between every 32 packed
+ * blocks.
* <li><b>Positions, Payloads, and Offsets</b>:
* <p>A position is an integer indicating where the term occurs within one document. A payload
* is a blob of metadata associated with current position. An offset is a pair of integers
@@ -102,10 +101,10 @@ import org.apache.lucene.util.packed.PackedInts;
* <ul>
* <li>PostingsHeader --> Header, PackedBlockSize
* <li>TermMetadata --> (DocFPDelta|SingletonDocID), PosFPDelta?, PosVIntBlockFPDelta?,
- * PayFPDelta?, SkipFPDelta?
+ * PayFPDelta?
* <li>Header, --> {@link CodecUtil#writeIndexHeader IndexHeader}
* <li>PackedBlockSize, SingletonDocID --> {@link DataOutput#writeVInt VInt}
- * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta, SkipFPDelta --> {@link
+ * <li>DocFPDelta, PosFPDelta, PayFPDelta, PosVIntBlockFPDelta --> {@link
* DataOutput#writeVLong VLong}
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}
* </ul>
@@ -135,9 +134,6 @@ import org.apache.lucene.util.packed.PackedInts;
* whether current block is packed format or VInt. When packed format, payloads and
* offsets are fetched from .pay, otherwise from .pos. (this value is neglected when
* total number of positions i.e. totalTermFreq is less or equal to PackedBlockSize).
- * <li>SkipFPDelta determines the position of this term's SkipData within the .doc file. In
- * particular, it is the length of the TermFreq data. SkipDelta is only stored if
- * DocFreq is not smaller than SkipMinimum (i.e. 128 in Lucene99PostingsFormat).
* <li>SingletonDocID is an optimization when a term only appears in one document. In this
* case, instead of writing a file pointer to the .doc file (DocFPDelta), and then a
* VIntBlock at that location, the single document ID is written to the term dictionary.
@@ -161,26 +157,19 @@ import org.apache.lucene.util.packed.PackedInts;
* IndexOptions#DOCS}). Skip data is saved at the end of each term's postings. The skip data
* is saved once for the entire postings list.
* <ul>
- * <li>docFile(.doc) --> Header, <TermFreqs, SkipData?><sup>TermCount</sup>, Footer
+ * <li>docFile(.doc) --> Header, <TermFreqs><sup>TermCount</sup>, Footer
* <li>Header --> {@link CodecUtil#writeIndexHeader IndexHeader}
- * <li>TermFreqs --> <PackedBlock> <sup>PackedDocBlockNum</sup>, VIntBlock?
- * <li>PackedBlock --> PackedDocDeltaBlock, PackedFreqBlock?
- * <li>VIntBlock --> <DocDelta[,
- * Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
- * <li>SkipData --> <<SkipLevelLength, SkipLevel> <sup>NumSkipLevels-1</sup>,
- * SkipLevel>, SkipDatum?
- * <li>SkipLevel --> <SkipDatum> <sup>TrimmedDocFreq/(PackedBlockSize^(Level +
- * 1))</sup>
- * <li>SkipDatum --> DocSkip, DocFPSkip, <PosFPSkip, PosBlockOffset, PayLength?,
- * PayFPSkip?>?, ImpactLength, <CompetitiveFreqDelta, CompetitiveNormDelta?>
- * <sup>ImpactCount</sup>, SkipChildLevelPointer?
+ * <li>TermFreqs --> <PackedBlock32> <sup>PackedDocBlockNum/32</sup>, VIntBlock?
+ * <li>PackedBlock32 --> Level1SkipData, <PackedBlock> <sup>32</sup>
+ * <li>PackedBlock --> Level0SkipData, PackedDocDeltaBlock, PackedFreqBlock?
+ * <li>VIntBlock -->
+ * <DocDelta[,Freq?]><sup>DocFreq-PackedBlockSize*PackedDocBlockNum</sup>
+ * <li>Level1SkipData --> DocDelta, DocFPDelta, Skip1NumBytes?, ImpactLength?, Impacts?,
+ * PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
+ * <li>Level0SkipData --> Skip0NumBytes, DocDelta, DocFPDelta, PackedBlockLength,
+ * ImpactLength?, Impacts?, PosFPDelta?, NextPosUpto?, PayFPDelta?, NextPayByteUpto?
* <li>PackedFreqBlock --> {@link PackedInts PackedInts}, uses patching
* <li>PackedDocDeltaBlock --> {@link PackedInts PackedInts}, does not use patching
- * <li>DocDelta, Freq, DocSkip, DocFPSkip, PosFPSkip, PosBlockOffset, PayByteUpto,
- * PayFPSkip, ImpactLength, CompetitiveFreqDelta --> {@link DataOutput#writeVInt
- * VInt}
- * <li>CompetitiveNormDelta --> {@link DataOutput#writeZLong ZLong}
- * <li>SkipChildLevelPointer --> {@link DataOutput#writeVLong VLong}
* <li>Footer --> {@link CodecUtil#writeFooter CodecFooter}
* </ul>
* <p>Notes:
@@ -212,24 +201,14 @@ import org.apache.lucene.util.packed.PackedInts;
* <p>7,4
* <li>PackedDocBlockNum is the number of packed blocks for current term's docids or
* frequencies. In particular, PackedDocBlockNum = floor(DocFreq/PackedBlockSize)
- * <li>TrimmedDocFreq = DocFreq % PackedBlockSize == 0 ? DocFreq - 1 : DocFreq. We use this
- * trick since the definition of skip entry is a little different from base interface.
- * In {@link MultiLevelSkipListWriter}, skip data is assumed to be saved for
- * skipInterval<sup>th</sup>, 2*skipInterval<sup>th</sup> ... posting in the list.
- * However, in Lucene99PostingsFormat, the skip data is saved for
- * skipInterval+1<sup>th</sup>, 2*skipInterval+1<sup>th</sup> ... posting
- * (skipInterval==PackedBlockSize in this case). When DocFreq is multiple of
- * PackedBlockSize, MultiLevelSkipListWriter will expect one more skip data than
- * Lucene99SkipWriter.
- * <li>SkipDatum is the metadata of one skip entry. For the first block (no matter packed or
- * VInt), it is omitted.
- * <li>DocSkip records the document number of every PackedBlockSize<sup>th</sup> document
- * number in the postings (i.e. last document number in each packed block). On disk it
- * is stored as the difference from previous value in the sequence.
- * <li>DocFPSkip records the file offsets of each block (excluding )posting at
- * PackedBlockSize+1<sup>th</sup>, 2*PackedBlockSize+1<sup>th</sup> ... , in DocFile.
- * The file offsets are relative to the start of current term's TermFreqs. On disk it is
- * also stored as the difference from previous SkipDatum in the sequence.
+ * <li>On skip data, DocDelta is the delta between the last doc of the previous block - or
+ * -1 if there is no previous block - and the last doc of this block. This helps know by
+ * how much the doc ID should be incremented in case the block gets skipped.
+ * <li>Skip0Length is the length of skip data at level 0. Encoding it is useful when skip
+ * data is never needed to quickly skip over skip data, e.g. if only using nextDoc(). It
+ * is also used when only the first fields of skip data are needed, in order to skip
+ * over remaining fields without reading them.
+ * <li>ImpactLength and Impacts are only stored if frequencies are indexed.
* <li>Since positions and payloads are also block encoded, the skip should skip to related
* block first, then fetch the values according to in-block offset. PosFPSkip and
* PayFPSkip record the file offsets of related block in .pos and .pay, respectively.
@@ -339,7 +318,10 @@ import org.apache.lucene.util.packed.PackedInts;
*
* @lucene.experimental
*/
-public final class Lucene99PostingsFormat extends PostingsFormat {
+public final class Lucene912PostingsFormat extends PostingsFormat {
+
+ /** Filename extension for some small metadata about how postings are encoded. */
+ public static final String META_EXTENSION = "psm";
/**
* Filename extension for document number, frequencies, and skip data. See chapter: <a
@@ -359,53 +341,52 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
/** Size of blocks. */
public static final int BLOCK_SIZE = ForUtil.BLOCK_SIZE;
- /**
- * Expert: The maximum number of skip levels. Smaller values result in slightly smaller indexes,
- * but slower skipping in big posting lists.
- */
- static final int MAX_SKIP_LEVELS = 10;
+ public static final int BLOCK_MASK = BLOCK_SIZE - 1;
+
+ /** We insert skip data on every block and every SKIP_FACTOR=32 blocks. */
+ public static final int LEVEL1_FACTOR = 32;
+
+ /** Total number of docs covered by level 1 skip data: 32 * 128 = 4,096 */
+ public static final int LEVEL1_NUM_DOCS = LEVEL1_FACTOR * BLOCK_SIZE;
+
+ public static final int LEVEL1_MASK = LEVEL1_NUM_DOCS - 1;
static final String TERMS_CODEC = "Lucene90PostingsWriterTerms";
- static final String DOC_CODEC = "Lucene99PostingsWriterDoc";
- static final String POS_CODEC = "Lucene99PostingsWriterPos";
- static final String PAY_CODEC = "Lucene99PostingsWriterPay";
+ static final String META_CODEC = "Lucene912PostingsWriterMeta";
+ static final String DOC_CODEC = "Lucene912PostingsWriterDoc";
+ static final String POS_CODEC = "Lucene912PostingsWriterPos";
+ static final String PAY_CODEC = "Lucene912PostingsWriterPay";
- // Increment version to change it
static final int VERSION_START = 0;
static final int VERSION_CURRENT = VERSION_START;
private final int minTermBlockSize;
private final int maxTermBlockSize;
- /** Creates {@code Lucene99PostingsFormat} with default settings. */
- public Lucene99PostingsFormat() {
+ /** Creates {@code Lucene912PostingsFormat} with default settings. */
+ public Lucene912PostingsFormat() {
this(
Lucene90BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE,
Lucene90BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
}
/**
- * Creates {@code Lucene99PostingsFormat} with custom values for {@code minBlockSize} and {@code
+ * Creates {@code Lucene912PostingsFormat} with custom values for {@code minBlockSize} and {@code
* maxBlockSize} passed to block terms dictionary.
*
* @see
* Lucene90BlockTreeTermsWriter#Lucene90BlockTreeTermsWriter(SegmentWriteState,PostingsWriterBase,int,int)
*/
- public Lucene99PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
- super("Lucene99");
+ public Lucene912PostingsFormat(int minTermBlockSize, int maxTermBlockSize) {
+ super("Lucene912");
Lucene90BlockTreeTermsWriter.validateSettings(minTermBlockSize, maxTermBlockSize);
this.minTermBlockSize = minTermBlockSize;
this.maxTermBlockSize = maxTermBlockSize;
}
- @Override
- public String toString() {
- return getName();
- }
-
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(state);
boolean success = false;
try {
FieldsConsumer ret =
@@ -422,7 +403,7 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene912PostingsReader(state);
boolean success = false;
try {
FieldsProducer ret = new Lucene90BlockTreeTermsReader(postingsReader, state);
@@ -436,7 +417,7 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
}
/**
- * Holds all state required for {@link Lucene99PostingsReader} to produce a {@link
+ * Holds all state required for {@link Lucene912PostingsReader} to produce a {@link
* org.apache.lucene.index.PostingsEnum} without re-seeking the terms dict.
*
* @lucene.internal
@@ -451,12 +432,6 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
/** file pointer to the start of the payloads enumeration, in {@link #PAY_EXTENSION} file */
public long payStartFP;
- /**
- * file offset for the start of the skip list, relative to docStartFP, if there are more than
- * {@link ForUtil#BLOCK_SIZE} docs; otherwise -1
- */
- public long skipOffset;
-
/**
* file offset for the last position in the last block, if there are more than {@link
* ForUtil#BLOCK_SIZE} positions; otherwise -1
@@ -477,7 +452,6 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
/** Sole constructor. */
public IntBlockTermState() {
- skipOffset = -1;
lastPosBlockOffset = -1;
singletonDocID = -1;
}
@@ -497,7 +471,6 @@ public final class Lucene99PostingsFormat extends PostingsFormat {
posStartFP = other.posStartFP;
payStartFP = other.payStartFP;
lastPosBlockOffset = other.lastPosBlockOffset;
- skipOffset = other.skipOffset;
singletonDocID = other.singletonDocID;
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsReader.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsReader.java
new file mode 100644
index 00000000000..5493bb588d5
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsReader.java
@@ -0,0 +1,2083 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.codecs.lucene912;
+
+import static org.apache.lucene.codecs.lucene912.ForUtil.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.LEVEL1_NUM_DOCS;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.META_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.POS_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.VERSION_START;
+
+import java.io.IOException;
+import java.util.AbstractList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.RandomAccess;
+
+import org.apache.lucene.codecs.BlockTermState;
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.codecs.PostingsReaderBase;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.IntBlockTermState;
+import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Impact;
+import org.apache.lucene.index.Impacts;
+import org.apache.lucene.index.ImpactsEnum;
+import org.apache.lucene.index.IndexFileNames;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.PostingsEnum;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.SlowImpactsEnum;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefBuilder;
+import org.apache.lucene.util.IOUtils;
+
+/**
+ * Concrete class that reads docId(maybe frq,pos,offset,payloads) list with postings format.
+ *
+ * @lucene.experimental
+ */
+public final class Lucene912PostingsReader extends PostingsReaderBase {
+
+ private final IndexInput docIn;
+ private final IndexInput posIn;
+ private final IndexInput payIn;
+
+ private final int maxNumImpactsAtLevel0;
+ private final int maxImpactNumBytesAtLevel0;
+ private final int maxNumImpactsAtLevel1;
+ private final int maxImpactNumBytesAtLevel1;
+
+ private final int version;
+
+ /** Sole constructor. */
+ public Lucene912PostingsReader(SegmentReadState state) throws IOException {
+ String metaName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.META_EXTENSION);
+ final long expectedDocFileLength, expectedPosFileLength, expectedPayFileLength;
+ ChecksumIndexInput metaIn = null;
+ boolean success = false;
+ try {
+ metaIn = state.directory.openChecksumInput(metaName, IOContext.READONCE);
+ version =
+ CodecUtil.checkIndexHeader(
+ metaIn,
+ META_CODEC,
+ VERSION_START,
+ VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ maxNumImpactsAtLevel0 = metaIn.readInt();
+ maxImpactNumBytesAtLevel0 = metaIn.readInt();
+ maxNumImpactsAtLevel1 = metaIn.readInt();
+ maxImpactNumBytesAtLevel1 = metaIn.readInt();
+ expectedDocFileLength = metaIn.readLong();
+ if (state.fieldInfos.hasProx()) {
+ expectedPosFileLength = metaIn.readLong();
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ expectedPayFileLength = metaIn.readLong();
+ } else {
+ expectedPayFileLength = -1;
+ }
+ } else {
+ expectedPosFileLength = -1;
+ expectedPayFileLength = -1;
+ }
+ CodecUtil.checkFooter(metaIn, null);
+ success = true;
+ } catch (Throwable t) {
+ if (metaIn != null) {
+ CodecUtil.checkFooter(metaIn, t);
+ throw new AssertionError("unreachable");
+ } else {
+ throw t;
+ }
+ } finally {
+ if (success) {
+ metaIn.close();
+ } else {
+ IOUtils.closeWhileHandlingException(metaIn);
+ }
+ }
+
+ success = false;
+ IndexInput docIn = null;
+ IndexInput posIn = null;
+ IndexInput payIn = null;
+
+ // NOTE: these data files are too costly to verify checksum against all the bytes on open,
+ // but for now we at least verify proper structure of the checksum footer: which looks
+ // for FOOTER_MAGIC + algorithmID. This is cheap and can detect some forms of corruption
+ // such as file truncation.
+
+ String docName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.DOC_EXTENSION);
+ try {
+ // Postings have a forward-only access pattern, so pass ReadAdvice.NORMAL to perform
+ // readahead.
+ docIn = state.directory.openInput(docName, state.context);
+ CodecUtil.checkIndexHeader(
+ docIn, DOC_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(docIn, expectedDocFileLength);
+
+ if (state.fieldInfos.hasProx()) {
+ String proxName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.POS_EXTENSION);
+ posIn = state.directory.openInput(proxName, state.context);
+ CodecUtil.checkIndexHeader(
+ posIn, POS_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(posIn, expectedPosFileLength);
+
+ if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
+ String payName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name,
+ state.segmentSuffix,
+ Lucene912PostingsFormat.PAY_EXTENSION);
+ payIn = state.directory.openInput(payName, state.context);
+ CodecUtil.checkIndexHeader(
+ payIn, PAY_CODEC, version, version, state.segmentInfo.getId(), state.segmentSuffix);
+ CodecUtil.retrieveChecksum(payIn, expectedPayFileLength);
+ }
+ }
+
+ this.docIn = docIn;
+ this.posIn = posIn;
+ this.payIn = payIn;
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(docIn, posIn, payIn);
+ }
+ }
+ }
+
+ @Override
+ public void init(IndexInput termsIn, SegmentReadState state) throws IOException {
+ // Make sure we are talking to the matching postings writer
+ CodecUtil.checkIndexHeader(
+ termsIn,
+ TERMS_CODEC,
+ VERSION_START,
+ VERSION_CURRENT,
+ state.segmentInfo.getId(),
+ state.segmentSuffix);
+ final int indexBlockSize = termsIn.readVInt();
+ if (indexBlockSize != BLOCK_SIZE) {
+ throw new IllegalStateException(
+ "index-time BLOCK_SIZE ("
+ + indexBlockSize
+ + ") != read-time BLOCK_SIZE ("
+ + BLOCK_SIZE
+ + ")");
+ }
+ }
+
+ static void prefixSum(long[] buffer, int count, long base) {
+ buffer[0] += base;
+ for (int i = 1; i < count; ++i) {
+ buffer[i] += buffer[i - 1];
+ }
+ }
+
+ static int findFirstGreater(long[] buffer, int target, int from) {
+ for (int i = from; i < BLOCK_SIZE; ++i) {
+ if (buffer[i] >= target) {
+ return i;
+ }
+ }
+ return BLOCK_SIZE;
+ }
+
+ @Override
+ public BlockTermState newTermState() {
+ return new IntBlockTermState();
+ }
+
+ @Override
+ public void close() throws IOException {
+ IOUtils.close(docIn, posIn, payIn);
+ }
+
+ @Override
+ public void decodeTerm(
+ DataInput in, FieldInfo fieldInfo, BlockTermState _termState, boolean absolute)
+ throws IOException {
+ final IntBlockTermState termState = (IntBlockTermState) _termState;
+ final boolean fieldHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ final boolean fieldHasOffsets =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ final boolean fieldHasPayloads = fieldInfo.hasPayloads();
+
+ if (absolute) {
+ termState.docStartFP = 0;
+ termState.posStartFP = 0;
+ termState.payStartFP = 0;
+ }
+
+ final long l = in.readVLong();
+ if ((l & 0x01) == 0) {
+ termState.docStartFP += l >>> 1;
+ if (termState.docFreq == 1) {
+ termState.singletonDocID = in.readVInt();
+ } else {
+ termState.singletonDocID = -1;
+ }
+ } else {
+ assert absolute == false;
+ assert termState.singletonDocID != -1;
+ termState.singletonDocID += BitUtil.zigZagDecode(l >>> 1);
+ }
+
+ if (fieldHasPositions) {
+ termState.posStartFP += in.readVLong();
+ if (fieldHasOffsets || fieldHasPayloads) {
+ termState.payStartFP += in.readVLong();
+ }
+ if (termState.totalTermFreq > BLOCK_SIZE) {
+ termState.lastPosBlockOffset = in.readVLong();
+ } else {
+ termState.lastPosBlockOffset = -1;
+ }
+ }
+ }
+
+ @Override
+ public PostingsEnum postings(
+ FieldInfo fieldInfo, BlockTermState termState, PostingsEnum reuse, int flags)
+ throws IOException {
+
+ boolean indexHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+
+ if (indexHasPositions == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
+ BlockDocsEnum docsEnum;
+ if (reuse instanceof BlockDocsEnum) {
+ docsEnum = (BlockDocsEnum) reuse;
+ if (!docsEnum.canReuse(docIn, fieldInfo)) {
+ docsEnum = new BlockDocsEnum(fieldInfo);
+ }
+ } else {
+ docsEnum = new BlockDocsEnum(fieldInfo);
+ }
+ return docsEnum.reset((IntBlockTermState) termState, flags);
+ } else {
+ EverythingEnum everythingEnum;
+ if (reuse instanceof EverythingEnum) {
+ everythingEnum = (EverythingEnum) reuse;
+ if (!everythingEnum.canReuse(docIn, fieldInfo)) {
+ everythingEnum = new EverythingEnum(fieldInfo);
+ }
+ } else {
+ everythingEnum = new EverythingEnum(fieldInfo);
+ }
+ return everythingEnum.reset((IntBlockTermState) termState, flags);
+ }
+ }
+
+ @Override
+ public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags)
+ throws IOException {
+ final boolean indexHasFreqs =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ final boolean indexHasPositions =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+
+ if (state.docFreq >= BLOCK_SIZE
+ && indexHasFreqs
+ && (indexHasPositions == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false)) {
+ return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state);
+ }
+
+ final boolean indexHasOffsets =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ final boolean indexHasPayloads = fieldInfo.hasPayloads();
+
+ if (state.docFreq >= BLOCK_SIZE
+ && indexHasPositions
+ && (indexHasOffsets == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false)
+ && (indexHasPayloads == false
+ || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {
+ return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);
+ }
+
+ return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));
+ }
+
+ final class BlockDocsEnum extends PostingsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+
+ private int docBufferUpto;
+
+ final IndexInput startDocIn;
+
+ IndexInput docIn;
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsetsOrPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)
+ private int docCountUpto; // number of docs in or before the current block
+ private int doc; // doc we last read
+ private long prevDocID; // last doc ID of the previous block
+
+ // level 0 skip data
+ private int level0LastDocID;
+ // level 1 skip data
+ private int level1LastDocID;
+ private long level1DocEndOffset;
+ private int level1DocCountUpto;
+
+ private boolean needsFreq; // true if the caller actually needs frequencies
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
+ private long freqFP;
+
+ public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
+ this.startDocIn = Lucene912PostingsReader.this.docIn;
+ this.docIn = null;
+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsetsOrPayloads =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0
+ || fieldInfo.hasPayloads();
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+ }
+
+ public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
+ return docIn == startDocIn
+ && indexHasFreq
+ == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0)
+ && indexHasPos
+ == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
+ >= 0)
+ && indexHasOffsetsOrPayloads
+ == (fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0
+ || fieldInfo.hasPayloads());
+ }
+
+ public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {
+ docFreq = termState.docFreq;
+ totalTermFreq = indexHasFreq ? termState.totalTermFreq : docFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+
+ doc = -1;
+ this.needsFreq = PostingsEnum.featureRequested(flags, PostingsEnum.FREQS);
+ if (indexHasFreq == false || needsFreq == false) {
+ // Filling this buffer may not be cheap when doing primary key lookups, so we make sure to
+ // not fill more than `docFreq` entries.
+ Arrays.fill(freqBuffer, 0, Math.min(ForUtil.BLOCK_SIZE, docFreq), 1);
+ }
+ prevDocID = -1;
+ docCountUpto = 0;
+ level0LastDocID = -1;
+ level1LastDocID = -1;
+ level1DocEndOffset = termState.docStartFP;
+ level1DocCountUpto = 0;
+ docBufferUpto = BLOCK_SIZE;
+ freqFP = -1;
+ return this;
+ }
+
+ @Override
+ public int freq() throws IOException {
+ if (freqFP != -1) {
+ docIn.seek(freqFP);
+ pforUtil.decode(docIn, freqBuffer);
+ }
+
+ return (int) freqBuffer[docBufferUpto - 1];
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public BytesRef getPayload() throws IOException {
+ return null;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillFullBlock() throws IOException {
+ assert docFreq - docCountUpto >= BLOCK_SIZE;
+
+ forDeltaUtil.decodeAndPrefixSum(docIn, prevDocID, docBuffer);
+
+ if (indexHasFreq) {
+ if (needsFreq) {
+ freqFP = docIn.getFilePointer();
+ }
+ pforUtil.skip(docIn);
+ }
+ docCountUpto += BLOCK_SIZE;
+ prevDocID = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ private void refillRemainder() throws IOException {
+ final int left = docFreq - docCountUpto;
+ assert left >= 0;
+ assert left < BLOCK_SIZE;
+
+ if (docFreq == 1) {
+ docBuffer[0] = singletonDocID;
+ freqBuffer[0] = totalTermFreq;
+ docBuffer[1] = NO_MORE_DOCS;
+ docCountUpto++;
+ } else {
+ // Read vInts:
+ PostingsUtil.readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq, needsFreq);
+ prefixSum(docBuffer, left, prevDocID);
+ docBuffer[left] = NO_MORE_DOCS;
+ docCountUpto += left;
+ }
+ docBufferUpto = 0;
+ freqFP = -1;
+ }
+
+ private void skipLevel1To(int target) throws IOException {
+ while (true) {
+ prevDocID = level1LastDocID;
+ level0LastDocID = level1LastDocID;
+ docIn.seek(level1DocEndOffset);
+ docCountUpto = level1DocCountUpto;
+ level1DocCountUpto += LEVEL1_NUM_DOCS;
+
+ if (docFreq - docCountUpto < LEVEL1_NUM_DOCS) {
+ level1LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+
+ level1LastDocID += docIn.readVInt();
+ level1DocEndOffset = docIn.readVLong() + docIn.getFilePointer();
+
+ if (level1LastDocID >= target) {
+ if (indexHasFreq) {
+ // skip impacts and pos skip data
+ docIn.skipBytes(docIn.readShort());
+ }
+ break;
+ }
+ }
+ }
+
+ private void skipLevel0To(int target) throws IOException {
+ while (true) {
+ prevDocID = level0LastDocID;
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ long skip0NumBytes = docIn.readVLong();
+ // end offset of skip data (before the actual data starts)
+ long skip0EndFP = docIn.getFilePointer() + skip0NumBytes;
+ int docDelta = readVInt15(docIn);
+ level0LastDocID += docDelta;
+
+ if (target <= level0LastDocID) {
+ docIn.seek(skip0EndFP);
+ break;
+ }
+
+ // skip block
+ docIn.skipBytes(readVLong15(docIn));
+ docCountUpto += BLOCK_SIZE;
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+ }
+ }
+
+ private void moveToNextLevel0Block() throws IOException {
+ if (doc == level1LastDocID) { // advance skip data on level 1
+ skipLevel1To(doc + 1);
+ }
+
+ prevDocID = level0LastDocID;
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ docIn.skipBytes(docIn.readVLong());
+ refillFullBlock();
+ level0LastDocID = (int) docBuffer[BLOCK_SIZE - 1];
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ refillRemainder();
+ }
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (doc == level0LastDocID) { // advance skip data on level 0
+ moveToNextLevel0Block();
+ }
+
+ return this.doc = (int) docBuffer[docBufferUpto++];
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > level0LastDocID) { // advance skip data on level 0
+
+ if (target > level1LastDocID) { // advance skip data on level 1
+ skipLevel1To(target);
+ }
+
+ skipLevel0To(target);
+
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ refillFullBlock();
+ } else {
+ refillRemainder();
+ }
+ }
+
+ int next = findFirstGreater(docBuffer, target, docBufferUpto);
+ this.doc = (int) docBuffer[next];
+ docBufferUpto = next + 1;
+ return doc;
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ final class EverythingEnum extends PostingsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
+
+ private final long[] payloadLengthBuffer;
+ private final long[] offsetStartDeltaBuffer;
+ private final long[] offsetLengthBuffer;
+
+ private byte[] payloadBytes;
+ private int payloadByteUpto;
+ private int payloadLength;
+
+ private int lastStartOffset;
+ private int startOffset;
+ private int endOffset;
+
+ private int docBufferUpto;
+ private int posBufferUpto;
+
+ final IndexInput startDocIn;
+
+ IndexInput docIn;
+ final IndexInput posIn;
+ final IndexInput payIn;
+ final BytesRef payload;
+
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+ final boolean indexHasOffsetsOrPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)
+ private int docCountUpto; // number of docs in or before the current block
+ private int doc; // doc we last read
+ private long prevDocID; // last doc ID of the previous block
+ private int freq; // freq we last read
+ private int position; // current position
+
+ // how many positions "behind" we are; nextPosition must
+ // skip these to "catch up":
+ private long posPendingCount;
+
+ // Where this term's postings start in the .pos file:
+ private long posTermStartFP;
+
+ // Where this term's payloads/offsets start in the .pay
+ // file:
+ private long payTermStartFP;
+
+ // File pointer where the last (vInt encoded) pos delta
+ // block is. We need this to know whether to bulk
+ // decode vs vInt decode the block:
+ private long lastPosBlockFP;
+
+ // level 0 skip data
+ private int level0LastDocID;
+ private long level0PosEndFP;
+ private int level0BlockPosUpto;
+ private long levelPayEndFP;
+ private int level0BlockPayUpto;
+ // level 1 skip data
+ private int level1LastDocID;
+ private long level1DocEndFP;
+ private int level1DocCountUpto;
+ private long level1PosEndFP;
+ private int level1BlockPosUpto;
+ private long level1PayEndFP;
+ private int level1BlockPayUpto;
+
+ private boolean needsOffsets; // true if we actually need offsets
+ private boolean needsPayloads; // true if we actually need payloads
+
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
+
+ public EverythingEnum(FieldInfo fieldInfo) throws IOException {
+ this.startDocIn = Lucene912PostingsReader.this.docIn;
+ this.docIn = null;
+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+ indexHasOffsetsOrPayloads = indexHasOffsets || indexHasPayloads;
+
+ this.posIn = Lucene912PostingsReader.this.posIn.clone();
+ if (indexHasOffsetsOrPayloads) {
+ this.payIn = Lucene912PostingsReader.this.payIn.clone();
+ } else {
+ this.payIn = null;
+ }
+ if (indexHasOffsets) {
+ offsetStartDeltaBuffer = new long[BLOCK_SIZE];
+ offsetLengthBuffer = new long[BLOCK_SIZE];
+ } else {
+ offsetStartDeltaBuffer = null;
+ offsetLengthBuffer = null;
+ startOffset = -1;
+ endOffset = -1;
+ }
+
+ if (indexHasPayloads) {
+ payloadLengthBuffer = new long[BLOCK_SIZE];
+ payloadBytes = new byte[128];
+ payload = new BytesRef();
+ } else {
+ payloadLengthBuffer = null;
+ payloadBytes = null;
+ payload = null;
+ }
+
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+ }
+
+ public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) {
+ return docIn == startDocIn
+ && indexHasOffsets
+ == (fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0)
+ && indexHasPayloads == fieldInfo.hasPayloads();
+ }
+
+ public PostingsEnum reset(IntBlockTermState termState, int flags) throws IOException {
+ docFreq = termState.docFreq;
+ posTermStartFP = termState.posStartFP;
+ payTermStartFP = termState.payStartFP;
+ totalTermFreq = termState.totalTermFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ posIn.seek(posTermStartFP);
+ if (indexHasOffsetsOrPayloads) {
+ payIn.seek(payTermStartFP);
+ }
+ level1PosEndFP = posTermStartFP;
+ level1PayEndFP = payTermStartFP;
+ level0PosEndFP = posTermStartFP;
+ levelPayEndFP = payTermStartFP;
+ posPendingCount = 0;
+ payloadByteUpto = 0;
+ if (termState.totalTermFreq < BLOCK_SIZE) {
+ lastPosBlockFP = posTermStartFP;
+ } else if (termState.totalTermFreq == BLOCK_SIZE) {
+ lastPosBlockFP = -1;
+ } else {
+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
+ }
+
+ this.needsOffsets = PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS);
+ this.needsPayloads = PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS);
+
+ doc = -1;
+ prevDocID = -1;
+ docCountUpto = 0;
+ level0LastDocID = -1;
+ level1LastDocID = -1;
+ level1DocEndFP = termState.docStartFP;
+ level1DocCountUpto = 0;
+ level1BlockPosUpto = 0;
+ level1BlockPayUpto = 0;
+ level0BlockPosUpto = 0;
+ level0BlockPayUpto = 0;
+ docBufferUpto = BLOCK_SIZE;
+ posBufferUpto = BLOCK_SIZE;
+ return this;
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return freq;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ final int left = docFreq - docCountUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, prevDocID, docBuffer);
+ pforUtil.decode(docIn, freqBuffer);
+ docCountUpto += BLOCK_SIZE;
+ } else if (docFreq == 1) {
+ docBuffer[0] = singletonDocID;
+ freqBuffer[0] = totalTermFreq;
+ docBuffer[1] = NO_MORE_DOCS;
+ docCountUpto++;
+ } else {
+ // Read vInts:
+ PostingsUtil.readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq, true);
+ prefixSum(docBuffer, left, prevDocID);
+ docBuffer[left] = NO_MORE_DOCS;
+ docCountUpto += left;
+ }
+ prevDocID = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ private void skipLevel1To(int target) throws IOException {
+ while (true) {
+ prevDocID = level1LastDocID;
+ level0LastDocID = level1LastDocID;
+ docIn.seek(level1DocEndFP);
+ level0PosEndFP = level1PosEndFP;
+ level0BlockPosUpto = level1BlockPosUpto;
+ if (indexHasOffsetsOrPayloads) {
+ levelPayEndFP = level1PayEndFP;
+ level0BlockPayUpto = level1BlockPayUpto;
+ }
+ docCountUpto = level1DocCountUpto;
+ level1DocCountUpto += LEVEL1_NUM_DOCS;
+
+ if (docFreq - docCountUpto < LEVEL1_NUM_DOCS) {
+ level1LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+
+ level1LastDocID += docIn.readVInt();
+ long delta = docIn.readVLong();
+ level1DocEndFP = delta + docIn.getFilePointer();
+
+ long skip1EndFP = docIn.readShort() + docIn.getFilePointer();
+ docIn.skipBytes(docIn.readShort()); // impacts
+ level1PosEndFP += docIn.readVLong();
+ level1BlockPosUpto = docIn.readByte();
+ if (indexHasOffsetsOrPayloads) {
+ level1PayEndFP += docIn.readVLong();
+ level1BlockPayUpto = docIn.readVInt();
+ }
+ assert docIn.getFilePointer() == skip1EndFP;
+
+ if (level1LastDocID >= target) {
+ break;
+ }
+ }
+ }
+
+ private void moveToNextLevel0Block() throws IOException {
+ if (doc == level1LastDocID) { // advance level 1 skip data
+ skipLevel1To(doc + 1);
+ }
+
+ // Now advance level 0 skip data
+ prevDocID = level0LastDocID;
+
+ assert docBufferUpto == BLOCK_SIZE;
+ if (level0PosEndFP >= posIn.getFilePointer()) {
+ posIn.seek(level0PosEndFP);
+ posPendingCount = level0BlockPosUpto;
+ if (indexHasOffsetsOrPayloads) {
+ assert levelPayEndFP >= payIn.getFilePointer();
+ payIn.seek(levelPayEndFP);
+ payloadByteUpto = level0BlockPayUpto;
+ }
+ posBufferUpto = BLOCK_SIZE;
+ }
+
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ docIn.readVLong(); // skip0 num bytes
+ int docDelta = readVInt15(docIn);
+ level0LastDocID += docDelta;
+ readVLong15(docIn); // block length
+ docIn.skipBytes(docIn.readVLong()); // impacts
+
+ level0PosEndFP += docIn.readVLong();
+ level0BlockPosUpto = docIn.readByte();
+ if (indexHasOffsetsOrPayloads) {
+ levelPayEndFP += docIn.readVLong();
+ level0BlockPayUpto = docIn.readVInt();
+ }
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ refillDocs();
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (doc == level0LastDocID) { // advance level 0 skip data
+ moveToNextLevel0Block();
+ }
+
+ this.doc = (int) docBuffer[docBufferUpto];
+ this.freq = (int) freqBuffer[docBufferUpto];
+ docBufferUpto++;
+ posPendingCount += freq;
+ position = 0;
+ lastStartOffset = 0;
+ return doc;
+ }
+
+ private void skipLevel0To(int target) throws IOException {
+ while (true) {
+ prevDocID = level0LastDocID;
+
+ // If nextBlockPosFP is less than the current FP, it means that the block of positions for
+ // the first docs of the next block are already decoded. In this case we just accumulate
+ // frequencies into posPendingCount instead of seeking backwards and decoding the same pos
+ // block again.
+ if (level0PosEndFP >= posIn.getFilePointer()) {
+ posIn.seek(level0PosEndFP);
+ posPendingCount = level0BlockPosUpto;
+ if (indexHasOffsetsOrPayloads) {
+ assert levelPayEndFP >= payIn.getFilePointer();
+ payIn.seek(levelPayEndFP);
+ payloadByteUpto = level0BlockPayUpto;
+ }
+ posBufferUpto = BLOCK_SIZE;
+ } else {
+ for (int i = docBufferUpto; i < BLOCK_SIZE; ++i) {
+ posPendingCount += freqBuffer[i];
+ }
+ }
+
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ docIn.readVLong(); // skip0 num bytes
+ int docDelta = readVInt15(docIn);
+ level0LastDocID += docDelta;
+
+ long blockLength = readVLong15(docIn);
+ long blockEndFP = docIn.getFilePointer() + blockLength;
+ docIn.skipBytes(docIn.readVLong()); // impacts
+
+ level0PosEndFP += docIn.readVLong();
+ level0BlockPosUpto = docIn.readByte();
+ if (indexHasOffsetsOrPayloads) {
+ levelPayEndFP += docIn.readVLong();
+ level0BlockPayUpto = docIn.readVInt();
+ }
+
+ if (target <= level0LastDocID) {
+ break;
+ }
+
+ docIn.seek(blockEndFP);
+ docCountUpto += BLOCK_SIZE;
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+ }
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > level0LastDocID) { // advance level 0 skip data
+
+ if (target > level1LastDocID) { // advance level 1 skip data
+ skipLevel1To(target);
+ }
+
+ skipLevel0To(target);
+
+ refillDocs();
+ }
+
+ int next = findFirstGreater(docBuffer, target, docBufferUpto);
+ for (int i = docBufferUpto; i <= next; ++i) {
+ posPendingCount += freqBuffer[i];
+ }
+ this.freq = (int) freqBuffer[next];
+ this.docBufferUpto = next + 1;
+ position = 0;
+ lastStartOffset = 0;
+
+ return this.doc = (int) docBuffer[next];
+ }
+
+ private void skipPositions() throws IOException {
+ // Skip positions now:
+ long toSkip = posPendingCount - freq;
+ // if (DEBUG) {
+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
+ // }
+
+ final int leftInBlock = BLOCK_SIZE - posBufferUpto;
+ if (toSkip < leftInBlock) {
+ int end = (int) (posBufferUpto + toSkip);
+ if (indexHasPayloads) {
+ for (int i = posBufferUpto; i < end; ++i) {
+ payloadByteUpto += payloadLengthBuffer[i];
+ }
+ }
+ posBufferUpto = end;
+ } else {
+ toSkip -= leftInBlock;
+ while (toSkip >= BLOCK_SIZE) {
+ assert posIn.getFilePointer() != lastPosBlockFP;
+ pforUtil.skip(posIn);
+
+ if (indexHasPayloads) {
+ // Skip payloadLength block:
+ pforUtil.skip(payIn);
+
+ // Skip payloadBytes block:
+ int numBytes = payIn.readVInt();
+ payIn.seek(payIn.getFilePointer() + numBytes);
+ }
+
+ if (indexHasOffsets) {
+ pforUtil.skip(payIn);
+ pforUtil.skip(payIn);
+ }
+ toSkip -= BLOCK_SIZE;
+ }
+ refillPositions();
+ payloadByteUpto = 0;
+ posBufferUpto = 0;
+ final int toSkipInt = (int) toSkip;
+ if (indexHasPayloads) {
+ for (int i = 0; i < toSkipInt; ++i) {
+ payloadByteUpto += payloadLengthBuffer[i];
+ }
+ }
+ posBufferUpto = toSkipInt;
+ }
+
+ position = 0;
+ lastStartOffset = 0;
+ }
+
+ private void refillPositions() throws IOException {
+ if (posIn.getFilePointer() == lastPosBlockFP) {
+ final int count = (int) (totalTermFreq % BLOCK_SIZE);
+ int payloadLength = 0;
+ int offsetLength = 0;
+ payloadByteUpto = 0;
+ for (int i = 0; i < count; i++) {
+ int code = posIn.readVInt();
+ if (indexHasPayloads) {
+ if ((code & 1) != 0) {
+ payloadLength = posIn.readVInt();
+ }
+ payloadLengthBuffer[i] = payloadLength;
+ posDeltaBuffer[i] = code >>> 1;
+ if (payloadLength != 0) {
+ if (payloadByteUpto + payloadLength > payloadBytes.length) {
+ payloadBytes = ArrayUtil.grow(payloadBytes, payloadByteUpto + payloadLength);
+ }
+ posIn.readBytes(payloadBytes, payloadByteUpto, payloadLength);
+ payloadByteUpto += payloadLength;
+ }
+ } else {
+ posDeltaBuffer[i] = code;
+ }
+
+ if (indexHasOffsets) {
+ int deltaCode = posIn.readVInt();
+ if ((deltaCode & 1) != 0) {
+ offsetLength = posIn.readVInt();
+ }
+ offsetStartDeltaBuffer[i] = deltaCode >>> 1;
+ offsetLengthBuffer[i] = offsetLength;
+ }
+ }
+ payloadByteUpto = 0;
+ } else {
+ pforUtil.decode(posIn, posDeltaBuffer);
+
+ if (indexHasPayloads) {
+ if (needsPayloads) {
+ pforUtil.decode(payIn, payloadLengthBuffer);
+ int numBytes = payIn.readVInt();
+
+ if (numBytes > payloadBytes.length) {
+ payloadBytes = ArrayUtil.growNoCopy(payloadBytes, numBytes);
+ }
+ payIn.readBytes(payloadBytes, 0, numBytes);
+ } else {
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ pforUtil.skip(payIn); // skip over lengths
+ int numBytes = payIn.readVInt(); // read length of payloadBytes
+ payIn.seek(payIn.getFilePointer() + numBytes); // skip over payloadBytes
+ }
+ payloadByteUpto = 0;
+ }
+
+ if (indexHasOffsets) {
+ if (needsOffsets) {
+ pforUtil.decode(payIn, offsetStartDeltaBuffer);
+ pforUtil.decode(payIn, offsetLengthBuffer);
+ } else {
+ // this works, because when writing a vint block we always force the first length to be
+ // written
+ pforUtil.skip(payIn); // skip over starts
+ pforUtil.skip(payIn); // skip over lengths
+ }
+ }
+ }
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ assert posPendingCount > 0;
+
+ if (posPendingCount > freq) {
+ skipPositions();
+ posPendingCount = freq;
+ }
+
+ if (posBufferUpto == BLOCK_SIZE) {
+ refillPositions();
+ posBufferUpto = 0;
+ }
+ position += posDeltaBuffer[posBufferUpto];
+
+ if (indexHasPayloads) {
+ payloadLength = (int) payloadLengthBuffer[posBufferUpto];
+ payload.bytes = payloadBytes;
+ payload.offset = payloadByteUpto;
+ payload.length = payloadLength;
+ payloadByteUpto += payloadLength;
+ }
+
+ if (indexHasOffsets) {
+ startOffset = lastStartOffset + (int) offsetStartDeltaBuffer[posBufferUpto];
+ endOffset = startOffset + (int) offsetLengthBuffer[posBufferUpto];
+ lastStartOffset = startOffset;
+ }
+
+ posBufferUpto++;
+ posPendingCount--;
+ return position;
+ }
+
+ @Override
+ public int startOffset() {
+ return startOffset;
+ }
+
+ @Override
+ public int endOffset() {
+ return endOffset;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ if (payloadLength == 0) {
+ return null;
+ } else {
+ return payload;
+ }
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ final class BlockImpactsDocsEnum extends ImpactsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+
+ private int docBufferUpto;
+
+ final IndexInput startDocIn;
+
+ IndexInput docIn;
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsetsOrPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private int docCountUpto; // number of docs in or before the current block
+ private int doc; // doc we last read
+ private long prevDocID; // last doc ID of the previous block
+ private long freqFP;
+
+ // true if we shallow-advanced to a new block that we have not decoded yet
+ private boolean needsRefilling;
+
+ // level 0 skip data
+ private int level0LastDocID;
+ private long level0DocEndFP;
+ private final BytesRef level0SerializedImpacts;
+ private final ByteArrayDataInput level0SerializedImpactsIn = new ByteArrayDataInput();
+ private final MutableImpactList level0Impacts;
+ // level 1 skip data
+ private int level1LastDocID;
+ private long level1DocEndFP;
+ private int level1DocCountUpto;
+ private final BytesRef level1SerializedImpacts;
+ private final ByteArrayDataInput level1SerializedImpactsIn = new ByteArrayDataInput();
+ private final MutableImpactList level1Impacts;
+
+ public BlockImpactsDocsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
+ throws IOException {
+ this.startDocIn = Lucene912PostingsReader.this.docIn;
+ this.docIn = null;
+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsetsOrPayloads =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0
+ || fieldInfo.hasPayloads();
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+
+ docFreq = termState.docFreq;
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+
+ doc = -1;
+ if (indexHasFreq == false) {
+ // Filling this buffer may not be cheap when doing primary key lookups, so we make sure to
+ // not fill more than `docFreq` entries.
+ Arrays.fill(freqBuffer, 0, Math.min(ForUtil.BLOCK_SIZE, docFreq), 1);
+ }
+ prevDocID = -1;
+ docCountUpto = 0;
+ level0LastDocID = -1;
+ level1LastDocID = -1;
+ level1DocEndFP = termState.docStartFP;
+ level1DocCountUpto = 0;
+ docBufferUpto = BLOCK_SIZE;
+ freqFP = -1;
+ level0SerializedImpacts = new BytesRef(maxImpactNumBytesAtLevel0);
+ level1SerializedImpacts = new BytesRef(maxImpactNumBytesAtLevel1);
+ level0Impacts = new MutableImpactList(maxNumImpactsAtLevel0);
+ level1Impacts = new MutableImpactList(maxNumImpactsAtLevel1);
+ }
+
+ @Override
+ public int freq() throws IOException {
+ if (freqFP != -1) {
+ docIn.seek(freqFP);
+ pforUtil.decode(docIn, freqBuffer);
+ freqFP = -1;
+ }
+ return (int) freqBuffer[docBufferUpto - 1];
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int startOffset() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() throws IOException {
+ return -1;
+ }
+
+ @Override
+ public BytesRef getPayload() throws IOException {
+ return null;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ final int left = docFreq - docCountUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, prevDocID, docBuffer);
+
+ if (indexHasFreq) {
+ freqFP = docIn.getFilePointer();
+ pforUtil.skip(docIn);
+ }
+ docCountUpto += BLOCK_SIZE;
+ } else {
+ // Read vInts:
+ PostingsUtil.readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq, true);
+ prefixSum(docBuffer, left, prevDocID);
+ docBuffer[left] = NO_MORE_DOCS;
+ freqFP = -1;
+ docCountUpto += left;
+ }
+ prevDocID = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ private void skipLevel1To(int target) throws IOException {
+ while (true) {
+ prevDocID = level1LastDocID;
+ level0LastDocID = level1LastDocID;
+ docIn.seek(level1DocEndFP);
+ docCountUpto = level1DocCountUpto;
+ level1DocCountUpto += LEVEL1_NUM_DOCS;
+
+ if (docFreq - docCountUpto < LEVEL1_NUM_DOCS) {
+ level1LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+
+ level1LastDocID += docIn.readVInt();
+ level1DocEndFP = docIn.readVLong() + docIn.getFilePointer();
+
+ if (level1LastDocID >= target) {
+ long skip1EndFP = docIn.readShort() + docIn.getFilePointer();
+ int numImpactBytes = docIn.readShort();
+ docIn.readBytes(level1SerializedImpacts.bytes, 0, numImpactBytes);
+ level1SerializedImpacts.length = numImpactBytes;
+ assert indexHasPos || docIn.getFilePointer() == skip1EndFP;
+ docIn.seek(skip1EndFP);
+ break;
+ }
+ }
+ }
+
+ private void skipLevel0To(int target) throws IOException {
+ while (true) {
+ prevDocID = level0LastDocID;
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ long skip0NumBytes = docIn.readVLong();
+ // end offset of skip data (before the actual data starts)
+ long skip0End = docIn.getFilePointer() + skip0NumBytes;
+ int docDelta = readVInt15(docIn);
+ long blockLength = readVLong15(docIn);
+
+ level0LastDocID += docDelta;
+
+ if (target <= level0LastDocID) {
+ level0DocEndFP = docIn.getFilePointer() + blockLength;
+ int numImpactBytes = docIn.readVInt();
+ docIn.readBytes(level0SerializedImpacts.bytes, 0, numImpactBytes);
+ level0SerializedImpacts.length = numImpactBytes;
+ docIn.seek(skip0End);
+ break;
+ }
+
+ // skip block
+ docIn.skipBytes(blockLength);
+ docCountUpto += BLOCK_SIZE;
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void advanceShallow(int target) throws IOException {
+ if (target > level0LastDocID) { // advance skip data on level 0
+ if (target > level1LastDocID) { // advance skip data on level 1
+ skipLevel1To(target);
+ } else if (needsRefilling) {
+ docIn.seek(level0DocEndFP);
+ docCountUpto += BLOCK_SIZE;
+ }
+
+ skipLevel0To(target);
+
+ needsRefilling = true;
+ }
+ }
+
+ private void moveToNextLevel0Block() throws IOException {
+ if (doc == level1LastDocID) {
+ skipLevel1To(doc + 1);
+ } else if (needsRefilling) {
+ docIn.seek(level0DocEndFP);
+ docCountUpto += BLOCK_SIZE;
+ }
+
+ prevDocID = level0LastDocID;
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ final long skip0Len = docIn.readVLong(); // skip len
+ final long skip0End = docIn.getFilePointer() + skip0Len;
+ final int docDelta = readVInt15(docIn);
+ final long blockLength = readVLong15(docIn);
+ level0LastDocID += docDelta;
+ level0DocEndFP = docIn.getFilePointer() + blockLength;
+ final int numImpactBytes = docIn.readVInt();
+ docIn.readBytes(level0SerializedImpacts.bytes, 0, numImpactBytes);
+ level0SerializedImpacts.length = numImpactBytes;
+ docIn.seek(skip0End);
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ }
+
+ refillDocs();
+ needsRefilling = false;
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ if (doc == level0LastDocID) {
+ moveToNextLevel0Block();
+ } else if (needsRefilling) {
+ refillDocs();
+ needsRefilling = false;
+ }
+
+ return this.doc = (int) docBuffer[docBufferUpto++];
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ if (target > level0LastDocID || needsRefilling) {
+ advanceShallow(target);
+ refillDocs();
+ needsRefilling = false;
+ }
+
+ int next = findFirstGreater(docBuffer, target, docBufferUpto);
+ this.doc = (int) docBuffer[next];
+ docBufferUpto = next + 1;
+ return doc;
+ }
+
+ @Override
+ public Impacts getImpacts() throws IOException {
+ return new Impacts() {
+
+ @Override
+ public int numLevels() {
+ int numLevels = 0;
+ if (level0LastDocID != NO_MORE_DOCS) {
+ numLevels++;
+ }
+ if (level1LastDocID != NO_MORE_DOCS) {
+ numLevels++;
+ }
+ if (numLevels == 0) {
+ numLevels++;
+ }
+ return numLevels;
+ }
+
+ @Override
+ public int getDocIdUpTo(int level) {
+ if (level0LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ return level0LastDocID;
+ }
+ level--;
+ }
+
+ if (level1LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ return level1LastDocID;
+ }
+ level--;
+ }
+
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public List<Impact> getImpacts(int level) {
+ if (level0LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ level0SerializedImpactsIn.reset(
+ level0SerializedImpacts.bytes, 0, level0SerializedImpacts.length);
+ readImpacts(level0SerializedImpactsIn, level0Impacts);
+ return level0Impacts;
+ }
+ level--;
+ }
+
+ if (level1LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ level1SerializedImpactsIn.reset(
+ level1SerializedImpacts.bytes, 0, level1SerializedImpacts.length);
+ readImpacts(level1SerializedImpactsIn, level1Impacts);
+ return level1Impacts;
+ }
+ level--;
+ }
+
+ return Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
+ }
+ };
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ final class BlockImpactsPostingsEnum extends ImpactsEnum {
+
+ final ForUtil forUtil = new ForUtil();
+ final ForDeltaUtil forDeltaUtil = new ForDeltaUtil(forUtil);
+ final PForUtil pforUtil = new PForUtil(forUtil);
+
+ private final long[] docBuffer = new long[BLOCK_SIZE + 1];
+ private final long[] freqBuffer = new long[BLOCK_SIZE];
+ private final long[] posDeltaBuffer = new long[BLOCK_SIZE];
+
+ private int docBufferUpto;
+ private int posBufferUpto;
+
+ final IndexInput startDocIn;
+
+ IndexInput docIn;
+ final IndexInput posIn;
+
+ final boolean indexHasFreq;
+ final boolean indexHasPos;
+ final boolean indexHasOffsets;
+ final boolean indexHasPayloads;
+ final boolean indexHasOffsetsOrPayloads;
+
+ private int docFreq; // number of docs in this posting list
+ private long totalTermFreq; // sum of freqBuffer in this posting list (or docFreq when omitted)
+ private int docCountUpto; // number of docs in or before the current block
+ private int doc; // doc we last read
+ private long prevDocID; // last doc ID of the previous block
+ private int freq; // freq we last read
+ private int position; // current position
+
+ // how many positions "behind" we are; nextPosition must
+ // skip these to "catch up":
+ private long posPendingCount;
+
+ // Where this term's postings start in the .pos file:
+ private long posTermStartFP;
+
+ // File pointer where the last (vInt encoded) pos delta
+ // block is. We need this to know whether to bulk
+ // decode vs vInt decode the block:
+ private long lastPosBlockFP;
+
+ // true if we shallow-advanced to a new block that we have not decoded yet
+ private boolean needsRefilling;
+
+ // level 0 skip data
+ private int level0LastDocID;
+ private long level0DocEndFP;
+ private long level0PosEndFP;
+ private int level0BlockPosUpto;
+ private final BytesRefBuilder level0SerializedImpacts = new BytesRefBuilder();
+ private final ByteArrayDataInput level0SerializedImpactsIn = new ByteArrayDataInput();
+ private final MutableImpactList level0Impacts;
+ // level 1 skip data
+ private int level1LastDocID;
+ private long level1DocEndFP;
+ private int level1DocCountUpto;
+ private long level1PosEndFP;
+ private int level1BlockPosUpto;
+ private final BytesRefBuilder level1SerializedImpacts = new BytesRefBuilder();
+ private final ByteArrayDataInput level1SerializedImpactsIn = new ByteArrayDataInput();
+ private final MutableImpactList level1Impacts;
+
+ private int singletonDocID; // docid when there is a single pulsed posting, otherwise -1
+
+ public BlockImpactsPostingsEnum(FieldInfo fieldInfo, IntBlockTermState termState)
+ throws IOException {
+ this.startDocIn = Lucene912PostingsReader.this.docIn;
+ this.docIn = null;
+ indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ indexHasPos =
+ fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
+ indexHasOffsets =
+ fieldInfo
+ .getIndexOptions()
+ .compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS)
+ >= 0;
+ indexHasPayloads = fieldInfo.hasPayloads();
+ indexHasOffsetsOrPayloads = indexHasOffsets || indexHasPayloads;
+
+ this.posIn = Lucene912PostingsReader.this.posIn.clone();
+
+ // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in
+ // advance()
+ docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
+
+ docFreq = termState.docFreq;
+ posTermStartFP = termState.posStartFP;
+ totalTermFreq = termState.totalTermFreq;
+ singletonDocID = termState.singletonDocID;
+ if (docIn == null) {
+ // lazy init
+ docIn = startDocIn.clone();
+ }
+ posIn.seek(posTermStartFP);
+ level1PosEndFP = posTermStartFP;
+ level0PosEndFP = posTermStartFP;
+ posPendingCount = 0;
+ if (termState.totalTermFreq < BLOCK_SIZE) {
+ lastPosBlockFP = posTermStartFP;
+ } else if (termState.totalTermFreq == BLOCK_SIZE) {
+ lastPosBlockFP = -1;
+ } else {
+ lastPosBlockFP = posTermStartFP + termState.lastPosBlockOffset;
+ }
+
+ doc = -1;
+ prevDocID = -1;
+ docCountUpto = 0;
+ level0LastDocID = -1;
+ level1LastDocID = -1;
+ level1DocEndFP = termState.docStartFP;
+ level1DocCountUpto = 0;
+ level1BlockPosUpto = 0;
+ docBufferUpto = BLOCK_SIZE;
+ posBufferUpto = BLOCK_SIZE;
+ level0SerializedImpacts.growNoCopy(maxImpactNumBytesAtLevel0);
+ level1SerializedImpacts.growNoCopy(maxImpactNumBytesAtLevel1);
+ level0Impacts = new MutableImpactList(maxNumImpactsAtLevel0);
+ level1Impacts = new MutableImpactList(maxNumImpactsAtLevel1);
+ }
+
+ @Override
+ public int freq() throws IOException {
+ return freq;
+ }
+
+ @Override
+ public int docID() {
+ return doc;
+ }
+
+ private void refillDocs() throws IOException {
+ final int left = docFreq - docCountUpto;
+ assert left >= 0;
+
+ if (left >= BLOCK_SIZE) {
+ forDeltaUtil.decodeAndPrefixSum(docIn, prevDocID, docBuffer);
+ pforUtil.decode(docIn, freqBuffer);
+ docCountUpto += BLOCK_SIZE;
+ } else if (docFreq == 1) {
+ docBuffer[0] = singletonDocID;
+ freqBuffer[0] = totalTermFreq;
+ docBuffer[1] = NO_MORE_DOCS;
+ docCountUpto++;
+ } else {
+ // Read vInts:
+ PostingsUtil.readVIntBlock(docIn, docBuffer, freqBuffer, left, indexHasFreq, true);
+ prefixSum(docBuffer, left, prevDocID);
+ docBuffer[left] = NO_MORE_DOCS;
+ docCountUpto += left;
+ }
+ prevDocID = docBuffer[BLOCK_SIZE - 1];
+ docBufferUpto = 0;
+ assert docBuffer[BLOCK_SIZE] == NO_MORE_DOCS;
+ }
+
+ private void skipLevel1To(int target) throws IOException {
+ while (true) {
+ prevDocID = level1LastDocID;
+ level0LastDocID = level1LastDocID;
+ docIn.seek(level1DocEndFP);
+ level0PosEndFP = level1PosEndFP;
+ level0BlockPosUpto = level1BlockPosUpto;
+ docCountUpto = level1DocCountUpto;
+ level1DocCountUpto += LEVEL1_NUM_DOCS;
+
+ if (docFreq - docCountUpto < LEVEL1_NUM_DOCS) {
+ level1LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+
+ level1LastDocID += docIn.readVInt();
+ level1DocEndFP = docIn.readVLong() + docIn.getFilePointer();
+
+ long skip1EndFP = docIn.readShort() + docIn.getFilePointer();
+ int numImpactBytes = docIn.readShort();
+ if (level1LastDocID >= target) {
+ docIn.readBytes(level1SerializedImpacts.bytes(), 0, numImpactBytes);
+ level1SerializedImpacts.setLength(numImpactBytes);
+ } else {
+ docIn.skipBytes(numImpactBytes);
+ }
+ level1PosEndFP += docIn.readVLong();
+ level1BlockPosUpto = docIn.readByte();
+ assert indexHasOffsetsOrPayloads || docIn.getFilePointer() == skip1EndFP;
+
+ if (level1LastDocID >= target) {
+ docIn.seek(skip1EndFP);
+ break;
+ }
+ }
+ }
+
+ private void skipLevel0To(int target) throws IOException {
+ while (true) {
+ prevDocID = level0LastDocID;
+
+ // If nextBlockPosFP is less than the current FP, it means that the block of positions for
+ // the first docs of the next block are already decoded. In this case we just accumulate
+ // frequencies into posPendingCount instead of seeking backwards and decoding the same pos
+ // block again.
+ if (level0PosEndFP >= posIn.getFilePointer()) {
+ posIn.seek(level0PosEndFP);
+ posPendingCount = level0BlockPosUpto;
+ posBufferUpto = BLOCK_SIZE;
+ } else {
+ for (int i = docBufferUpto; i < BLOCK_SIZE; ++i) {
+ posPendingCount += freqBuffer[i];
+ }
+ }
+
+ if (docFreq - docCountUpto >= BLOCK_SIZE) {
+ docIn.readVLong(); // skip0 num bytes
+ int docDelta = readVInt15(docIn);
+ long blockLength = readVLong15(docIn);
+ level0DocEndFP = docIn.getFilePointer() + blockLength;
+
+ level0LastDocID += docDelta;
+
+ if (target <= level0LastDocID) {
+ int numImpactBytes = docIn.readVInt();
+ docIn.readBytes(level0SerializedImpacts.bytes(), 0, numImpactBytes);
+ level0SerializedImpacts.setLength(numImpactBytes);
+ level0PosEndFP += docIn.readVLong();
+ level0BlockPosUpto = docIn.readByte();
+ if (indexHasOffsetsOrPayloads) {
+ docIn.readVLong(); // pay fp delta
+ docIn.readVInt(); // pay upto
+ }
+ break;
+ }
+ // skip block
+ docIn.skipBytes(docIn.readVLong()); // impacts
+ level0PosEndFP += docIn.readVLong();
+ level0BlockPosUpto = docIn.readVInt();
+ docIn.seek(level0DocEndFP);
+ docCountUpto += BLOCK_SIZE;
+ } else {
+ level0LastDocID = DocIdSetIterator.NO_MORE_DOCS;
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void advanceShallow(int target) throws IOException {
+ if (target > level0LastDocID) { // advance level 0 skip data
+
+ if (target > level1LastDocID) { // advance skip data on level 1
+ skipLevel1To(target);
+ } else if (needsRefilling) {
+ docIn.seek(level0DocEndFP);
+ docCountUpto += BLOCK_SIZE;
+ }
+
+ skipLevel0To(target);
+
+ needsRefilling = true;
+ }
+ }
+
+ @Override
+ public Impacts getImpacts() throws IOException {
+ return new Impacts() {
+
+ @Override
+ public int numLevels() {
+ int numLevels = 0;
+ if (level0LastDocID != NO_MORE_DOCS) {
+ numLevels++;
+ }
+ if (level1LastDocID != NO_MORE_DOCS) {
+ numLevels++;
+ }
+ if (numLevels == 0) {
+ numLevels++;
+ }
+ return numLevels;
+ }
+
+ @Override
+ public int getDocIdUpTo(int level) {
+ if (level0LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ return level0LastDocID;
+ }
+ level--;
+ }
+
+ if (level1LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ return level1LastDocID;
+ }
+ level--;
+ }
+
+ return NO_MORE_DOCS;
+ }
+
+ @Override
+ public List<Impact> getImpacts(int level) {
+ if (level0LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ level0SerializedImpactsIn.reset(
+ level0SerializedImpacts.bytes(), 0, level0SerializedImpacts.length());
+ readImpacts(level0SerializedImpactsIn, level0Impacts);
+ return level0Impacts;
+ }
+ level--;
+ }
+
+ if (level1LastDocID != NO_MORE_DOCS) {
+ if (level == 0) {
+ level1SerializedImpactsIn.reset(
+ level1SerializedImpacts.bytes(), 0, level1SerializedImpacts.length());
+ readImpacts(level1SerializedImpactsIn, level1Impacts);
+ return level1Impacts;
+ }
+ level--;
+ }
+
+ return Collections.singletonList(new Impact(Integer.MAX_VALUE, 1L));
+ }
+ };
+ }
+
+ @Override
+ public int nextDoc() throws IOException {
+ advanceShallow(doc + 1);
+ if (needsRefilling) {
+ refillDocs();
+ needsRefilling = false;
+ }
+
+ doc = (int) docBuffer[docBufferUpto];
+ freq = (int) freqBuffer[docBufferUpto];
+ posPendingCount += freq;
+ docBufferUpto++;
+ position = 0;
+ return this.doc;
+ }
+
+ @Override
+ public int advance(int target) throws IOException {
+ advanceShallow(target);
+ if (needsRefilling) {
+ refillDocs();
+ needsRefilling = false;
+ }
+
+ int next = findFirstGreater(docBuffer, target, docBufferUpto);
+ for (int i = docBufferUpto; i <= next; ++i) {
+ posPendingCount += freqBuffer[i];
+ }
+ freq = (int) freqBuffer[next];
+ docBufferUpto = next + 1;
+ position = 0;
+ return this.doc = (int) docBuffer[next];
+ }
+
+ private void skipPositions() throws IOException {
+ // Skip positions now:
+ long toSkip = posPendingCount - freq;
+ // if (DEBUG) {
+ // System.out.println(" FPR.skipPositions: toSkip=" + toSkip);
+ // }
+
+ final int leftInBlock = BLOCK_SIZE - posBufferUpto;
+ if (toSkip < leftInBlock) {
+ posBufferUpto += toSkip;
+ } else {
+ toSkip -= leftInBlock;
+ while (toSkip >= BLOCK_SIZE) {
+ assert posIn.getFilePointer() != lastPosBlockFP;
+ pforUtil.skip(posIn);
+ toSkip -= BLOCK_SIZE;
+ }
+ refillPositions();
+ posBufferUpto = (int) toSkip;
+ }
+
+ position = 0;
+ }
+
+ private void refillPositions() throws IOException {
+ if (posIn.getFilePointer() == lastPosBlockFP) {
+ final int count = (int) (totalTermFreq % BLOCK_SIZE);
+ int payloadLength = 0;
+ for (int i = 0; i < count; i++) {
+ int code = posIn.readVInt();
+ if (indexHasPayloads) {
+ if ((code & 1) != 0) {
+ payloadLength = posIn.readVInt();
+ }
+ posDeltaBuffer[i] = code >>> 1;
+ if (payloadLength != 0) {
+ posIn.skipBytes(payloadLength);
+ }
+ } else {
+ posDeltaBuffer[i] = code;
+ }
+
+ if (indexHasOffsets) {
+ int deltaCode = posIn.readVInt();
+ if ((deltaCode & 1) != 0) {
+ posIn.readVInt(); // offset length
+ }
+ }
+ }
+ } else {
+ pforUtil.decode(posIn, posDeltaBuffer);
+ }
+ }
+
+ @Override
+ public int nextPosition() throws IOException {
+ assert posPendingCount > 0;
+
+ if (posPendingCount > freq) {
+ skipPositions();
+ posPendingCount = freq;
+ }
+
+ if (posBufferUpto == BLOCK_SIZE) {
+ refillPositions();
+ posBufferUpto = 0;
+ }
+ position += posDeltaBuffer[posBufferUpto];
+
+ posBufferUpto++;
+ posPendingCount--;
+ return position;
+ }
+
+ @Override
+ public int startOffset() {
+ return -1;
+ }
+
+ @Override
+ public int endOffset() {
+ return -1;
+ }
+
+ @Override
+ public BytesRef getPayload() {
+ return null;
+ }
+
+ @Override
+ public long cost() {
+ return docFreq;
+ }
+ }
+
+ /**
+ * @see Lucene912PostingsWriter#writeVInt15(org.apache.lucene.store.DataOutput, int)
+ */
+ static int readVInt15(DataInput in) throws IOException {
+ short s = in.readShort();
+ if (s >= 0) {
+ return s;
+ } else {
+ return (s & 0x7FFF) | (in.readVInt() << 15);
+ }
+ }
+
+ /**
+ * @see Lucene912PostingsWriter#writeVLong15(org.apache.lucene.store.DataOutput, long)
+ */
+ static long readVLong15(DataInput in) throws IOException {
+ short s = in.readShort();
+ if (s >= 0) {
+ return s;
+ } else {
+ return (s & 0x7FFFL) | (in.readVLong() << 15);
+ }
+ }
+
+ static class MutableImpactList extends AbstractList<Impact> implements RandomAccess {
+ int length;
+ final Impact[] impacts;
+
+ MutableImpactList(int capacity) {
+ impacts = new Impact[capacity];
+ for (int i = 0; i < capacity; ++i) {
+ impacts[i] = new Impact(Integer.MAX_VALUE, 1L);
+ }
+ }
+
+ @Override
+ public Impact get(int index) {
+ return impacts[index];
+ }
+
+ @Override
+ public int size() {
+ return length;
+ }
+ }
+
+ static MutableImpactList readImpacts(ByteArrayDataInput in, MutableImpactList reuse) {
+ int freq = 0;
+ long norm = 0;
+ int length = 0;
+ while (in.getPosition() < in.length()) {
+ int freqDelta = in.readVInt();
+ if ((freqDelta & 0x01) != 0) {
+ freq += 1 + (freqDelta >>> 1);
+ try {
+ norm += 1 + in.readZLong();
+ } catch (IOException e) {
+ throw new RuntimeException(e); // cannot happen on a BADI
+ }
+ } else {
+ freq += 1 + (freqDelta >>> 1);
+ norm++;
+ }
+ Impact impact = reuse.impacts[length];
+ impact.freq = freq;
+ impact.norm = norm;
+ length++;
+ }
+ reuse.length = length;
+ return reuse;
+ }
+
+ @Override
+ public void checkIntegrity() throws IOException {
+ if (docIn != null) {
+ CodecUtil.checksumEntireFile(docIn);
+ }
+ if (posIn != null) {
+ CodecUtil.checksumEntireFile(posIn);
+ }
+ if (payIn != null) {
+ CodecUtil.checksumEntireFile(payIn);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return getClass().getSimpleName()
+ + "(positions="
+ + (posIn != null)
+ + ",payloads="
+ + (payIn != null)
+ + ")";
+ }
+}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsWriter.java
similarity index 55%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsWriter.java
index 61949cfe227..b307080b215 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/Lucene99PostingsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/Lucene912PostingsWriter.java
@@ -14,28 +14,31 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
-import static org.apache.lucene.codecs.lucene99.ForUtil.BLOCK_SIZE;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.DOC_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.MAX_SKIP_LEVELS;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.PAY_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.POS_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.TERMS_CODEC;
-import static org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.VERSION_CURRENT;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.*;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.BLOCK_SIZE;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.DOC_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.PAY_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.TERMS_CODEC;
+import static org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.VERSION_CURRENT;
import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
import org.apache.lucene.codecs.BlockTermState;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.PushPostingsWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat.IntBlockTermState;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat.IntBlockTermState;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.Impact;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.ArrayUtil;
@@ -43,21 +46,16 @@ import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
-/**
- * Concrete class that writes docId(maybe frq,pos,offset,payloads) list with postings format.
- *
- * <p>Postings list for each term will be stored separately.
- *
- * @see Lucene99SkipWriter for details about skipping setting and postings layout.
- * @lucene.experimental
- */
-public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
+/** Writer for {@link Lucene912PostingsFormat}. */
+public class Lucene912PostingsWriter extends PushPostingsWriterBase {
+ static final IntBlockTermState EMPTY_STATE = new IntBlockTermState();
+
+ IndexOutput metaOut;
IndexOutput docOut;
IndexOutput posOut;
IndexOutput payOut;
- static final IntBlockTermState emptyState = new IntBlockTermState();
IntBlockTermState lastState;
// Holds starting file pointers for current term:
@@ -78,12 +76,15 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
private byte[] payloadBytes;
private int payloadByteUpto;
- private int lastBlockDocID;
- private long lastBlockPosFP;
- private long lastBlockPayFP;
- private int lastBlockPosBufferUpto;
- private int lastBlockPayloadByteUpto;
+ private int level0LastDocID;
+ private long level0LastPosFP;
+ private long level0LastPayFP;
+
+ private int level1LastDocID;
+ private long level1LastPosFP;
+ private long level1LastPayFP;
+ private int docID;
private int lastDocID;
private int lastPosition;
private int lastStartOffset;
@@ -91,24 +92,52 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
private final PForUtil pforUtil;
private final ForDeltaUtil forDeltaUtil;
- private final Lucene99SkipWriter skipWriter;
private boolean fieldHasNorms;
private NumericDocValues norms;
- private final CompetitiveImpactAccumulator competitiveFreqNormAccumulator =
+ private final CompetitiveImpactAccumulator level0FreqNormAccumulator =
+ new CompetitiveImpactAccumulator();
+ private final CompetitiveImpactAccumulator level1CompetitiveFreqNormAccumulator =
new CompetitiveImpactAccumulator();
- /** Creates a postings writer */
- public Lucene99PostingsWriter(SegmentWriteState state) throws IOException {
-
+ private int maxNumImpactsAtLevel0;
+ private int maxImpactNumBytesAtLevel0;
+ private int maxNumImpactsAtLevel1;
+ private int maxImpactNumBytesAtLevel1;
+
+ /** Scratch output that we use to be able to prepend the encoded length, e.g. impacts. */
+ private final ByteBuffersDataOutput scratchOutput = ByteBuffersDataOutput.newResettableInstance();
+
+ /**
+ * Output for a single block. This is useful to be able to prepend skip data before each block,
+ * which can only be computed once the block is encoded. The content is then typically copied to
+ * {@link #level1Output}.
+ */
+ private final ByteBuffersDataOutput level0Output = ByteBuffersDataOutput.newResettableInstance();
+
+ /**
+ * Output for groups of 32 blocks. This is useful to prepend skip data for these 32 blocks, which
+ * can only be done once we have encoded these 32 blocks. The content is then typically copied to
+ * {@link #docCount}.
+ */
+ private final ByteBuffersDataOutput level1Output = ByteBuffersDataOutput.newResettableInstance();
+
+ /** Sole constructor. */
+ public Lucene912PostingsWriter(SegmentWriteState state) throws IOException {
+ String metaFileName =
+ IndexFileNames.segmentFileName(
+ state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.META_EXTENSION);
String docFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.DOC_EXTENSION);
- docOut = state.directory.createOutput(docFileName, state.context);
+ state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.DOC_EXTENSION);
+ metaOut = state.directory.createOutput(metaFileName, state.context);
IndexOutput posOut = null;
IndexOutput payOut = null;
boolean success = false;
try {
+ docOut = state.directory.createOutput(docFileName, state.context);
+ CodecUtil.writeIndexHeader(
+ metaOut, META_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
CodecUtil.writeIndexHeader(
docOut, DOC_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
final ForUtil forUtil = new ForUtil();
@@ -118,7 +147,7 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
posDeltaBuffer = new long[BLOCK_SIZE];
String posFileName =
IndexFileNames.segmentFileName(
- state.segmentInfo.name, state.segmentSuffix, Lucene99PostingsFormat.POS_EXTENSION);
+ state.segmentInfo.name, state.segmentSuffix, Lucene912PostingsFormat.POS_EXTENSION);
posOut = state.directory.createOutput(posFileName, state.context);
CodecUtil.writeIndexHeader(
posOut, POS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
@@ -144,7 +173,7 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
IndexFileNames.segmentFileName(
state.segmentInfo.name,
state.segmentSuffix,
- Lucene99PostingsFormat.PAY_EXTENSION);
+ Lucene912PostingsFormat.PAY_EXTENSION);
payOut = state.directory.createOutput(payFileName, state.context);
CodecUtil.writeIndexHeader(
payOut, PAY_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
@@ -161,17 +190,12 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
success = true;
} finally {
if (!success) {
- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
+ IOUtils.closeWhileHandlingException(metaOut, docOut, posOut, payOut);
}
}
docDeltaBuffer = new long[BLOCK_SIZE];
freqBuffer = new long[BLOCK_SIZE];
-
- // TODO: should we try skipping every 2/4 blocks...?
- skipWriter =
- new Lucene99SkipWriter(
- MAX_SKIP_LEVELS, BLOCK_SIZE, state.segmentInfo.maxDoc(), docOut, posOut, payOut);
}
@Override
@@ -189,8 +213,7 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
@Override
public void setField(FieldInfo fieldInfo) {
super.setField(fieldInfo);
- skipWriter.setField(writePositions, writeOffsets, writePayloads);
- lastState = emptyState;
+ lastState = EMPTY_STATE;
fieldHasNorms = fieldInfo.hasNorms();
}
@@ -199,37 +222,31 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
docStartFP = docOut.getFilePointer();
if (writePositions) {
posStartFP = posOut.getFilePointer();
+ level1LastPosFP = level0LastPosFP = posStartFP;
if (writePayloads || writeOffsets) {
payStartFP = payOut.getFilePointer();
+ level1LastPayFP = level0LastPayFP = payStartFP;
}
}
- lastDocID = 0;
- lastBlockDocID = -1;
- skipWriter.resetSkip();
+ lastDocID = -1;
+ level0LastDocID = -1;
+ level1LastDocID = -1;
this.norms = norms;
- competitiveFreqNormAccumulator.clear();
+ if (writeFreqs) {
+ level0FreqNormAccumulator.clear();
+ }
}
@Override
public void startDoc(int docID, int termDocFreq) throws IOException {
- // Have collected a block of docs, and get a new doc.
- // Should write skip data as well as postings list for
- // current block.
- if (lastBlockDocID != -1 && docBufferUpto == 0) {
- skipWriter.bufferSkip(
- lastBlockDocID,
- competitiveFreqNormAccumulator,
- docCount,
- lastBlockPosFP,
- lastBlockPayFP,
- lastBlockPosBufferUpto,
- lastBlockPayloadByteUpto);
- competitiveFreqNormAccumulator.clear();
+ if (docBufferUpto == BLOCK_SIZE) {
+ flushDocBlock(false);
+ docBufferUpto = 0;
}
final int docDelta = docID - lastDocID;
- if (docID < 0 || (docCount > 0 && docDelta <= 0)) {
+ if (docID < 0 || docDelta <= 0) {
throw new CorruptIndexException(
"docs out of order (" + docID + " <= " + lastDocID + " )", docOut);
}
@@ -239,40 +256,29 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
freqBuffer[docBufferUpto] = termDocFreq;
}
- docBufferUpto++;
- docCount++;
-
- if (docBufferUpto == BLOCK_SIZE) {
- forDeltaUtil.encodeDeltas(docDeltaBuffer, docOut);
- if (writeFreqs) {
- pforUtil.encode(freqBuffer, docOut);
- }
- // NOTE: don't set docBufferUpto back to 0 here;
- // finishDoc will do so (because it needs to see that
- // the block was filled so it can save skip data)
- }
-
- lastDocID = docID;
+ this.docID = docID;
lastPosition = 0;
lastStartOffset = 0;
- long norm;
- if (fieldHasNorms) {
- boolean found = norms.advanceExact(docID);
- if (found == false) {
- // This can happen if indexing hits a problem after adding a doc to the
- // postings but before buffering the norm. Such documents are written
- // deleted and will go away on the first merge.
- norm = 1L;
+ if (writeFreqs) {
+ long norm;
+ if (fieldHasNorms) {
+ boolean found = norms.advanceExact(docID);
+ if (found == false) {
+ // This can happen if indexing hits a problem after adding a doc to the
+ // postings but before buffering the norm. Such documents are written
+ // deleted and will go away on the first merge.
+ norm = 1L;
+ } else {
+ norm = norms.longValue();
+ assert norm != 0 : docID;
+ }
} else {
- norm = norms.longValue();
- assert norm != 0 : docID;
+ norm = 1L;
}
- } else {
- norm = 1L;
- }
- competitiveFreqNormAccumulator.add(writeFreqs ? termDocFreq : 1, norm);
+ level0FreqNormAccumulator.add(termDocFreq, norm);
+ }
}
@Override
@@ -335,20 +341,163 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
@Override
public void finishDoc() throws IOException {
- // Since we don't know df for current term, we had to buffer
- // those skip data for each block, and when a new doc comes,
- // write them to skip file.
- if (docBufferUpto == BLOCK_SIZE) {
- lastBlockDocID = lastDocID;
- if (posOut != null) {
- if (payOut != null) {
- lastBlockPayFP = payOut.getFilePointer();
+ docBufferUpto++;
+ docCount++;
+
+ lastDocID = docID;
+ }
+
+ /**
+ * Special vints that are encoded on 2 bytes if they require 15 bits or less. VInt becomes
+ * especially slow when the number of bytes is variable, so this special layout helps in the case
+ * when the number likely requires 15 bits or less
+ */
+ static void writeVInt15(DataOutput out, int v) throws IOException {
+ assert v >= 0;
+ writeVLong15(out, v);
+ }
+
+ /**
+ * @see #writeVInt15(DataOutput, int)
+ */
+ static void writeVLong15(DataOutput out, long v) throws IOException {
+ assert v >= 0;
+ if ((v & ~0x7FFFL) == 0) {
+ out.writeShort((short) v);
+ } else {
+ out.writeShort((short) (0x8000 | (v & 0x7FFF)));
+ out.writeVLong(v >> 15);
+ }
+ }
+
+ private void flushDocBlock(boolean finishTerm) throws IOException {
+ assert docBufferUpto != 0;
+
+ if (docBufferUpto < BLOCK_SIZE) {
+ assert finishTerm;
+ PostingsUtil.writeVIntBlock(
+ level0Output, docDeltaBuffer, freqBuffer, docBufferUpto, writeFreqs);
+ } else {
+ if (writeFreqs) {
+ List<Impact> impacts = level0FreqNormAccumulator.getCompetitiveFreqNormPairs();
+ if (impacts.size() > maxNumImpactsAtLevel0) {
+ maxNumImpactsAtLevel0 = impacts.size();
+ }
+ writeImpacts(impacts, scratchOutput);
+ assert level0Output.size() == 0;
+ if (scratchOutput.size() > maxImpactNumBytesAtLevel0) {
+ maxImpactNumBytesAtLevel0 = Math.toIntExact(scratchOutput.size());
+ }
+ level0Output.writeVLong(scratchOutput.size());
+ scratchOutput.copyTo(level0Output);
+ scratchOutput.reset();
+ if (writePositions) {
+ level0Output.writeVLong(posOut.getFilePointer() - level0LastPosFP);
+ level0Output.writeByte((byte) posBufferUpto);
+ level0LastPosFP = posOut.getFilePointer();
+
+ if (writeOffsets || writePayloads) {
+ level0Output.writeVLong(payOut.getFilePointer() - level0LastPayFP);
+ level0Output.writeVInt(payloadByteUpto);
+ level0LastPayFP = payOut.getFilePointer();
+ }
}
- lastBlockPosFP = posOut.getFilePointer();
- lastBlockPosBufferUpto = posBufferUpto;
- lastBlockPayloadByteUpto = payloadByteUpto;
}
- docBufferUpto = 0;
+ long numSkipBytes = level0Output.size();
+ forDeltaUtil.encodeDeltas(docDeltaBuffer, level0Output);
+ if (writeFreqs) {
+ pforUtil.encode(freqBuffer, level0Output);
+ }
+
+ // docID - lastBlockDocID is at least 128, so it can never fit a single byte with a vint
+ // Even if we subtracted 128, only extremely dense blocks would be eligible to a single byte
+ // so let's go with 2 bytes right away
+ writeVInt15(scratchOutput, docID - level0LastDocID);
+ writeVLong15(scratchOutput, level0Output.size());
+ numSkipBytes += scratchOutput.size();
+ level1Output.writeVLong(numSkipBytes);
+ scratchOutput.copyTo(level1Output);
+ scratchOutput.reset();
+ }
+
+ level0Output.copyTo(level1Output);
+ level0Output.reset();
+ level0LastDocID = docID;
+ if (writeFreqs) {
+ level1CompetitiveFreqNormAccumulator.addAll(level0FreqNormAccumulator);
+ level0FreqNormAccumulator.clear();
+ }
+
+ if ((docCount & LEVEL1_MASK) == 0) { // true every 32 blocks (4,096 docs)
+ writeLevel1SkipData();
+ level1LastDocID = docID;
+ level1CompetitiveFreqNormAccumulator.clear();
+ } else if (finishTerm) {
+ level1Output.copyTo(docOut);
+ level1Output.reset();
+ level1CompetitiveFreqNormAccumulator.clear();
+ }
+ }
+
+ private void writeLevel1SkipData() throws IOException {
+ docOut.writeVInt(docID - level1LastDocID);
+ long numImpactBytes = scratchOutput.size();
+ final long level1End;
+ if (writeFreqs) {
+ List<Impact> impacts = level1CompetitiveFreqNormAccumulator.getCompetitiveFreqNormPairs();
+ if (impacts.size() > maxNumImpactsAtLevel1) {
+ maxNumImpactsAtLevel1 = impacts.size();
+ }
+ writeImpacts(impacts, scratchOutput);
+ numImpactBytes = scratchOutput.size();
+ if (numImpactBytes > maxImpactNumBytesAtLevel1) {
+ maxImpactNumBytesAtLevel1 = Math.toIntExact(numImpactBytes);
+ }
+ if (writePositions) {
+ scratchOutput.writeVLong(posOut.getFilePointer() - level1LastPosFP);
+ scratchOutput.writeByte((byte) posBufferUpto);
+ level1LastPosFP = posOut.getFilePointer();
+ if (writeOffsets || writePayloads) {
+ scratchOutput.writeVLong(payOut.getFilePointer() - level1LastPayFP);
+ scratchOutput.writeVInt(payloadByteUpto);
+ level1LastPayFP = payOut.getFilePointer();
+ }
+ }
+ final long level1Len = 2 * Short.BYTES + scratchOutput.size() + level1Output.size();
+ docOut.writeVLong(level1Len);
+ level1End = docOut.getFilePointer() + level1Len;
+ // There are at most 128 impacts, that require at most 2 bytes each
+ assert numImpactBytes <= Short.MAX_VALUE;
+ // Like impacts plus a few vlongs, still way under the max short value
+ assert scratchOutput.size() + Short.BYTES <= Short.MAX_VALUE;
+ docOut.writeShort((short) (scratchOutput.size() + Short.BYTES));
+ docOut.writeShort((short) numImpactBytes);
+ scratchOutput.copyTo(docOut);
+ scratchOutput.reset();
+ } else {
+ docOut.writeVLong(level1Output.size());
+ level1End = docOut.getFilePointer() + level1Output.size();
+ }
+ level1Output.copyTo(docOut);
+ level1Output.reset();
+ assert docOut.getFilePointer() == level1End : docOut.getFilePointer() + " " + level1End;
+ }
+
+ static void writeImpacts(Collection<Impact> impacts, DataOutput out) throws IOException {
+ Impact previous = new Impact(0, 0);
+ for (Impact impact : impacts) {
+ assert impact.freq > previous.freq;
+ assert Long.compareUnsigned(impact.norm, previous.norm) > 0;
+ int freqDelta = impact.freq - previous.freq - 1;
+ long normDelta = impact.norm - previous.norm - 1;
+ if (normDelta == 0) {
+ // most of time, norm only increases by 1, so we can fold everything in a single byte
+ out.writeVInt(freqDelta << 1);
+ } else {
+ out.writeVInt((freqDelta << 1) | 1);
+ out.writeZLong(normDelta);
+ }
+ previous = impact;
}
}
@@ -367,11 +516,10 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
final int singletonDocID;
if (state.docFreq == 1) {
// pulse the singleton docid into the term dictionary, freq is implicitly totalTermFreq
- singletonDocID = (int) docDeltaBuffer[0];
+ singletonDocID = (int) docDeltaBuffer[0] - 1;
} else {
singletonDocID = -1;
- // Group vInt encode the remaining doc deltas and freqs:
- PostingsUtil.writeVIntBlock(docOut, docDeltaBuffer, freqBuffer, docBufferUpto, writeFreqs);
+ flushDocBlock(true);
}
final long lastPosBlockOffset;
@@ -387,6 +535,7 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
lastPosBlockOffset = -1;
}
if (posBufferUpto > 0) {
+ assert posBufferUpto < BLOCK_SIZE;
// TODO: should we send offsets/payloads to
// .pay...? seems wasteful (have to store extra
// vLong for low (< BLOCK_SIZE) DF terms = vast vast
@@ -438,22 +587,15 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
lastPosBlockOffset = -1;
}
- long skipOffset;
- if (docCount > BLOCK_SIZE) {
- skipOffset = skipWriter.writeSkip(docOut) - docStartFP;
- } else {
- skipOffset = -1;
- }
-
state.docStartFP = docStartFP;
state.posStartFP = posStartFP;
state.payStartFP = payStartFP;
state.singletonDocID = singletonDocID;
- state.skipOffset = skipOffset;
+
state.lastPosBlockOffset = lastPosBlockOffset;
docBufferUpto = 0;
posBufferUpto = 0;
- lastDocID = 0;
+ lastDocID = -1;
docCount = 0;
}
@@ -463,7 +605,7 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
throws IOException {
IntBlockTermState state = (IntBlockTermState) _state;
if (absolute) {
- lastState = emptyState;
+ lastState = EMPTY_STATE;
assert lastState.docStartFP == 0;
}
@@ -495,9 +637,6 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
out.writeVLong(state.lastPosBlockOffset);
}
}
- if (state.skipOffset != -1) {
- out.writeVLong(state.skipOffset);
- }
lastState = state;
}
@@ -515,14 +654,28 @@ public final class Lucene99PostingsWriter extends PushPostingsWriterBase {
if (payOut != null) {
CodecUtil.writeFooter(payOut);
}
+ if (metaOut != null) {
+ metaOut.writeInt(maxNumImpactsAtLevel0);
+ metaOut.writeInt(maxImpactNumBytesAtLevel0);
+ metaOut.writeInt(maxNumImpactsAtLevel1);
+ metaOut.writeInt(maxImpactNumBytesAtLevel1);
+ metaOut.writeLong(docOut.getFilePointer());
+ if (posOut != null) {
+ metaOut.writeLong(posOut.getFilePointer());
+ if (payOut != null) {
+ metaOut.writeLong(payOut.getFilePointer());
+ }
+ }
+ CodecUtil.writeFooter(metaOut);
+ }
success = true;
} finally {
if (success) {
- IOUtils.close(docOut, posOut, payOut);
+ IOUtils.close(metaOut, docOut, posOut, payOut);
} else {
- IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
+ IOUtils.closeWhileHandlingException(metaOut, docOut, posOut, payOut);
}
- docOut = posOut = payOut = null;
+ metaOut = docOut = posOut = payOut = null;
}
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PForUtil.java
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/PForUtil.java
index 93ebb3d0f9d..c0ddac6f3ec 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PForUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PForUtil.java
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import java.util.Arrays;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PostingsUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java
similarity index 92%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/PostingsUtil.java
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java
index 678754047b6..4834dd73e22 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/PostingsUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java
@@ -14,11 +14,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.IndexOutput;
/** Utility class to encode/decode postings block. */
final class PostingsUtil {
@@ -53,7 +53,7 @@ final class PostingsUtil {
/** Write freq buffer with variable-length encoding and doc buffer with group-varint encoding. */
static void writeVIntBlock(
- IndexOutput docOut, long[] docBuffer, long[] freqBuffer, int num, boolean writeFreqs)
+ DataOutput docOut, long[] docBuffer, long[] freqBuffer, int num, boolean writeFreqs)
throws IOException {
if (writeFreqs) {
for (int i = 0; i < num; i++) {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/gen_ForUtil.py
similarity index 99%
rename from lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py
rename to lucene/core/src/java/org/apache/lucene/codecs/lucene912/gen_ForUtil.py
index cd81d493570..c6a33ceef53 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/gen_ForUtil.py
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/gen_ForUtil.py
@@ -40,7 +40,7 @@ HEADER = """// This file has been automatically generated, DO NOT EDIT
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/package-info.java
similarity index 95%
copy from lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
copy to lucene/core/src/java/org/apache/lucene/codecs/lucene912/package-info.java
index 2df2cfc9a30..853f86a855a 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/package-info.java
@@ -16,7 +16,7 @@
*/
/**
- * Lucene 9.9 file format.
+ * Lucene 9.12 file format.
*
* <h2>Apache Lucene - Index File Formats</h2>
*
@@ -151,15 +151,15 @@
* field names. These are used to store auxiliary information about the document, such as its
* title, url, or an identifier to access a database. The set of stored fields are what is
* returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term dictionary}. A
+ * <li>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Term dictionary}. A
* dictionary containing all of the terms used in all of the indexed fields of all of the
* documents. The dictionary also contains the number of documents which contain the term, and
* pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Frequency data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Term Frequency data}. For
* each term in the dictionary, the numbers of all the documents that contain that term, and
* the frequency of the term in that document, unless frequencies are omitted ({@link
* org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Proximity data}. For
+ * <li>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Term Proximity data}. For
* each term in the dictionary, the positions that the term occurs in each document. Note that
* this will not exist if all fields in all documents omit position data.
* <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
@@ -255,27 +255,27 @@
* <td>The stored fields for documents</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Dictionary}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Term Dictionary}</td>
* <td>.tim</td>
* <td>The term dictionary, stores term info</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Index}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Term Index}</td>
* <td>.tip</td>
* <td>The index into the Term Dictionary</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Frequencies}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Frequencies}</td>
* <td>.doc</td>
* <td>Contains the list of docs which contain each term along with frequency</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Positions}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Positions}</td>
* <td>.pos</td>
* <td>Stores position information about where a term occurs in the index</td>
* </tr>
* <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Payloads}</td>
+ * <td>{@link org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat Payloads}</td>
* <td>.pay</td>
* <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
* </tr>
@@ -411,6 +411,9 @@
* layer and not writing the node ids for the zeroth layer.
* <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector
* format to utilize int8 quantized vectors for float32 vector search.
+ * <li>In version 9.12, skip data was refactored to have only two levels: every 128 docs and every
+ * 4,06 docs, and to be inlined in postings lists. This resulted in a speedup for queries that
+ * need skipping, especially conjunctions.
* </ul>
*
* <a id="Limitations"></a>
@@ -425,4 +428,4 @@
* <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
* VInt} values which have no limit. </div>
*/
-package org.apache.lucene.codecs.lucene99;
+package org.apache.lucene.codecs.lucene912;
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
index 2df2cfc9a30..98a029853e7 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene99/package-info.java
@@ -15,414 +15,5 @@
* limitations under the License.
*/
-/**
- * Lucene 9.9 file format.
- *
- * <h2>Apache Lucene - Index File Formats</h2>
- *
- * <div>
- *
- * <ul>
- * <li><a href="#Introduction">Introduction</a>
- * <li><a href="#Definitions">Definitions</a>
- * <ul>
- * <li><a href="#Inverted_Indexing">Inverted Indexing</a>
- * <li><a href="#Types_of_Fields">Types of Fields</a>
- * <li><a href="#Segments">Segments</a>
- * <li><a href="#Document_Numbers">Document Numbers</a>
- * </ul>
- * <li><a href="#Overview">Index Structure Overview</a>
- * <li><a href="#File_Naming">File Naming</a>
- * <li><a href="#file-names">Summary of File Extensions</a>
- * <ul>
- * <li><a href="#Lock_File">Lock File</a>
- * <li><a href="#History">History</a>
- * <li><a href="#Limitations">Limitations</a>
- * </ul>
- * </ul>
- *
- * </div> <a id="Introduction"></a>
- *
- * <h3>Introduction</h3>
- *
- * <div>
- *
- * <p>This document defines the index file formats used in this version of Lucene. If you are using
- * a different version of Lucene, please consult the copy of <code>docs/</code> that was distributed
- * with the version you are using.
- *
- * <p>This document attempts to provide a high-level definition of the Apache Lucene file formats.
- * </div> <a id="Definitions"></a>
- *
- * <h3>Definitions</h3>
- *
- * <div>
- *
- * <p>The fundamental concepts in Lucene are index, document, field and term.
- *
- * <p>An index contains a sequence of documents.
- *
- * <ul>
- * <li>A document is a sequence of fields.
- * <li>A field is a named sequence of terms.
- * <li>A term is a sequence of bytes.
- * </ul>
- *
- * <p>The same sequence of bytes in two different fields is considered a different term. Thus terms
- * are represented as a pair: the string naming the field, and the bytes within the field. <a
- * id="Inverted_Indexing"></a>
- *
- * <h4>Inverted Indexing</h4>
- *
- * <p>Lucene's index stores terms and statistics about those terms in order to make term-based
- * search more efficient. Lucene's terms index falls into the family of indexes known as an
- * <i>inverted index.</i> This is because it can list, for a term, the documents that contain it.
- * This is the inverse of the natural relationship, in which documents list terms. <a
- * id="Types_of_Fields"></a>
- *
- * <h4>Types of Fields</h4>
- *
- * <p>In Lucene, fields may be <i>stored</i>, in which case their text is stored in the index
- * literally, in a non-inverted manner. Fields that are inverted are called <i>indexed</i>. A field
- * may be both stored and indexed.
- *
- * <p>The text of a field may be <i>tokenized</i> into terms to be indexed, or the text of a field
- * may be used literally as a term to be indexed. Most fields are tokenized, but sometimes it is
- * useful for certain identifier fields to be indexed literally.
- *
- * <p>See the {@link org.apache.lucene.document.Field Field} java docs for more information on
- * Fields. <a id="Segments"></a>
- *
- * <h4>Segments</h4>
- *
- * <p>Lucene indexes may be composed of multiple sub-indexes, or <i>segments</i>. Each segment is a
- * fully independent index, which could be searched separately. Indexes evolve by:
- *
- * <ol>
- * <li>Creating new segments for newly added documents.
- * <li>Merging existing segments.
- * </ol>
- *
- * <p>Searches may involve multiple segments and/or multiple indexes, each index potentially
- * composed of a set of segments. <a id="Document_Numbers"></a>
- *
- * <h4>Document Numbers</h4>
- *
- * <p>Internally, Lucene refers to documents by an integer <i>document number</i>. The first
- * document added to an index is numbered zero, and each subsequent document added gets a number one
- * greater than the previous.
- *
- * <p>Note that a document's number may change, so caution should be taken when storing these
- * numbers outside of Lucene. In particular, numbers may change in the following situations:
- *
- * <ul>
- * <li>
- * <p>The numbers stored in each segment are unique only within the segment, and must be
- * converted before they can be used in a larger context. The standard technique is to
- * allocate each segment a range of values, based on the range of numbers used in that
- * segment. To convert a document number from a segment to an external value, the segment's
- * <i>base</i> document number is added. To convert an external value back to a
- * segment-specific value, the segment is identified by the range that the external value is
- * in, and the segment's base value is subtracted. For example two five document segments
- * might be combined, so that the first segment has a base value of zero, and the second of
- * five. Document three from the second segment would have an external value of eight.
- * <li>
- * <p>When documents are deleted, gaps are created in the numbering. These are eventually
- * removed as the index evolves through merging. Deleted documents are dropped when segments
- * are merged. A freshly-merged segment thus has no gaps in its numbering.
- * </ul>
- *
- * </div> <a id="Overview"></a>
- *
- * <h3>Index Structure Overview</h3>
- *
- * <div>
- *
- * <p>Each segment index maintains the following:
- *
- * <ul>
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment info}. This
- * contains metadata about a segment, such as the number of documents, what files it uses, and
- * information about how the segment is sorted
- * <li>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Field names}. This
- * contains metadata about the set of named fields used in the index.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Stored Field values}.
- * This contains, for each document, a list of attribute-value pairs, where the attributes are
- * field names. These are used to store auxiliary information about the document, such as its
- * title, url, or an identifier to access a database. The set of stored fields are what is
- * returned for each hit when searching. This is keyed by document number.
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term dictionary}. A
- * dictionary containing all of the terms used in all of the indexed fields of all of the
- * documents. The dictionary also contains the number of documents which contain the term, and
- * pointers to the term's frequency and proximity data.
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Frequency data}. For
- * each term in the dictionary, the numbers of all the documents that contain that term, and
- * the frequency of the term in that document, unless frequencies are omitted ({@link
- * org.apache.lucene.index.IndexOptions#DOCS IndexOptions.DOCS})
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Proximity data}. For
- * each term in the dictionary, the positions that the term occurs in each document. Note that
- * this will not exist if all fields in all documents omit position data.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Normalization factors}. For
- * each field in each document, a value is stored that is multiplied into the score for hits
- * on that field.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vectors}. For each
- * field in each document, the term vector (sometimes called document vector) may be stored. A
- * term vector consists of term text and term frequency. To add Term Vectors to your index see
- * the {@link org.apache.lucene.document.Field Field} constructors
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-document values}. Like
- * stored values, these are also keyed by document number, but are generally intended to be
- * loaded into main memory for fast access. Whereas stored values are generally intended for
- * summary results from searches, per-document values are useful for things like scoring
- * factors.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live documents}. An
- * optional file indicating which documents are live.
- * <li>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}. Optional pair
- * of files, recording dimensionally indexed fields, to enable fast numeric range filtering
- * and large numeric values like BigInteger and BigDecimal (1D) and geographic shape
- * intersection (2D, 3D).
- * <li>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}. The
- * vector format stores numeric vectors in a format optimized for random access and
- * computation, supporting high-dimensional nearest-neighbor search.
- * </ul>
- *
- * <p>Details on each of these are provided in their linked pages. </div> <a id="File_Naming"></a>
- *
- * <h3>File Naming</h3>
- *
- * <div>
- *
- * <p>All files belonging to a segment have the same name with varying extensions. The extensions
- * correspond to the different file formats described below. When using the Compound File format
- * (default for small segments) these files (except for the Segment info file, the Lock file, and
- * Deleted documents file) are collapsed into a single .cfs file (see below for details)
- *
- * <p>Typically, all segments in an index are stored in a single directory, although this is not
- * required.
- *
- * <p>File names are never re-used. That is, when any file is saved to the Directory it is given a
- * never before used filename. This is achieved using a simple generations approach. For example,
- * the first segments file is segments_1, then segments_2, etc. The generation is a sequential long
- * integer represented in alpha-numeric (base 36) form. </div> <a id="file-names"></a>
- *
- * <h3>Summary of File Extensions</h3>
- *
- * <div>
- *
- * <p>The following table summarizes the names and extensions of the files in Lucene:
- *
- * <table class="padding4" style="border-spacing: 1px; border-collapse: separate">
- * <caption>lucene filenames by extension</caption>
- * <tr>
- * <th>Name</th>
- * <th>Extension</th>
- * <th>Brief Description</th>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.index.SegmentInfos Segments File}</td>
- * <td>segments_N</td>
- * <td>Stores information about a commit point</td>
- * </tr>
- * <tr>
- * <td><a href="#Lock_File">Lock File</a></td>
- * <td>write.lock</td>
- * <td>The Write lock prevents multiple IndexWriters from writing to the same
- * file.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99SegmentInfoFormat Segment Info}</td>
- * <td>.si</td>
- * <td>Stores metadata about a segment</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90CompoundFormat Compound File}</td>
- * <td>.cfs, .cfe</td>
- * <td>An optional "virtual" file consisting of all the other index files for
- * systems that frequently run out of file handles.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene94.Lucene94FieldInfosFormat Fields}</td>
- * <td>.fnm</td>
- * <td>Stores information about the fields</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Index}</td>
- * <td>.fdx</td>
- * <td>Contains pointers to field data</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat Field Data}</td>
- * <td>.fdt</td>
- * <td>The stored fields for documents</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Dictionary}</td>
- * <td>.tim</td>
- * <td>The term dictionary, stores term info</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Term Index}</td>
- * <td>.tip</td>
- * <td>The index into the Term Dictionary</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Frequencies}</td>
- * <td>.doc</td>
- * <td>Contains the list of docs which contain each term along with frequency</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Positions}</td>
- * <td>.pos</td>
- * <td>Stores position information about where a term occurs in the index</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat Payloads}</td>
- * <td>.pay</td>
- * <td>Stores additional per-position metadata information such as character offsets and user payloads</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90NormsFormat Norms}</td>
- * <td>.nvd, .nvm</td>
- * <td>Encodes length and boost factors for docs and fields</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat Per-Document Values}</td>
- * <td>.dvd, .dvm</td>
- * <td>Encodes additional scoring factors or other per-document information.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Index}</td>
- * <td>.tvx</td>
- * <td>Stores offset into the document data file</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90TermVectorsFormat Term Vector Data}</td>
- * <td>.tvd</td>
- * <td>Contains term vector data.</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90LiveDocsFormat Live Documents}</td>
- * <td>.liv</td>
- * <td>Info about what documents are live</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene90.Lucene90PointsFormat Point values}</td>
- * <td>.dii, .dim</td>
- * <td>Holds indexed points</td>
- * </tr>
- * <tr>
- * <td>{@link org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat Vector values}</td>
- * <td>.vec, .vem, .veq, vex</td>
- * <td>Holds indexed vectors; <code>.vec</code> files contain the raw vector data,
- * <code>.vem</code> the vector metadata, <code>.veq</code> the quantized vector data, and <code>.vex</code> the
- * hnsw graph data.</td>
- * </tr>
- * </table>
- *
- * </div> <a id="Lock_File"></a>
- *
- * <h3>Lock File</h3>
- *
- * The write lock, which is stored in the index directory by default, is named "write.lock". If the
- * lock directory is different from the index directory then the write lock will be named
- * "XXXX-write.lock" where XXXX is a unique prefix derived from the full path to the index
- * directory. When this file is present, a writer is currently modifying the index (adding or
- * removing documents). This lock file ensures that only one writer is modifying the index at a
- * time. <a id="History"></a>
- *
- * <h3>History</h3>
- *
- * <p>Compatibility notes are provided in this document, describing how file formats have changed
- * from prior versions:
- *
- * <ul>
- * <li>In version 2.1, the file format was changed to allow lock-less commits (ie, no more commit
- * lock). The change is fully backwards compatible: you can open a pre-2.1 index for searching
- * or adding/deleting of docs. When the new segments file is saved (committed), it will be
- * written in the new file format (meaning no specific "upgrade" process is needed). But note
- * that once a commit has occurred, pre-2.1 Lucene will not be able to read the index.
- * <li>In version 2.3, the file format was changed to allow segments to share a single set of doc
- * store (vectors & stored fields) files. This allows for faster indexing in certain
- * cases. The change is fully backwards compatible (in the same way as the lock-less commits
- * change in 2.1).
- * <li>In version 2.4, Strings are now written as true UTF-8 byte sequence, not Java's modified
- * UTF-8. See <a href="http://issues.apache.org/jira/browse/LUCENE-510">LUCENE-510</a> for
- * details.
- * <li>In version 2.9, an optional opaque Map<String,String> CommitUserData may be passed to
- * IndexWriter's commit methods (and later retrieved), which is recorded in the segments_N
- * file. See <a href="http://issues.apache.org/jira/browse/LUCENE-1382">LUCENE-1382</a> for
- * details. Also, diagnostics were added to each segment written recording details about why
- * it was written (due to flush, merge; which OS/JRE was used; etc.). See issue <a
- * href="http://issues.apache.org/jira/browse/LUCENE-1654">LUCENE-1654</a> for details.
- * <li>In version 3.0, compressed fields are no longer written to the index (they can still be
- * read, but on merge the new segment will write them, uncompressed). See issue <a
- * href="http://issues.apache.org/jira/browse/LUCENE-1960">LUCENE-1960</a> for details.
- * <li>In version 3.1, segments records the code version that created them. See <a
- * href="http://issues.apache.org/jira/browse/LUCENE-2720">LUCENE-2720</a> for details.
- * Additionally segments track explicitly whether or not they have term vectors. See <a
- * href="http://issues.apache.org/jira/browse/LUCENE-2811">LUCENE-2811</a> for details.
- * <li>In version 3.2, numeric fields are written as natively to stored fields file, previously
- * they were stored in text format only.
- * <li>In version 3.4, fields can omit position data while still indexing term frequencies.
- * <li>In version 4.0, the format of the inverted index became extensible via the {@link
- * org.apache.lucene.codecs.Codec Codec} api. Fast per-document storage ({@code DocValues})
- * was introduced. Normalization factors need no longer be a single byte, they can be any
- * {@link org.apache.lucene.index.NumericDocValues NumericDocValues}. Terms need not be
- * unicode strings, they can be any byte sequence. Term offsets can optionally be indexed into
- * the postings lists. Payloads can be stored in the term vectors.
- * <li>In version 4.1, the format of the postings list changed to use either of FOR compression or
- * variable-byte encoding, depending upon the frequency of the term. Terms appearing only once
- * were changed to inline directly into the term dictionary. Stored fields are compressed by
- * default.
- * <li>In version 4.2, term vectors are compressed by default. DocValues has a new multi-valued
- * type (SortedSet), that can be used for faceting/grouping/joining on multi-valued fields.
- * <li>In version 4.5, DocValues were extended to explicitly represent missing values.
- * <li>In version 4.6, FieldInfos were extended to support per-field DocValues generation, to
- * allow updating NumericDocValues fields.
- * <li>In version 4.8, checksum footers were added to the end of each index file for improved data
- * integrity. Specifically, the last 8 bytes of every index file contain the zlib-crc32
- * checksum of the file.
- * <li>In version 4.9, DocValues has a new multi-valued numeric type (SortedNumeric) that is
- * suitable for faceting/sorting/analytics.
- * <li>In version 5.4, DocValues have been improved to store more information on disk: addresses
- * for binary fields and ord indexes for multi-valued fields.
- * <li>In version 6.0, Points were added, for multi-dimensional range/distance search.
- * <li>In version 6.2, new Segment info format that reads/writes the index sort, to support index
- * sorting.
- * <li>In version 7.0, DocValues have been improved to better support sparse doc values thanks to
- * an iterator API.
- * <li>In version 8.0, postings have been enhanced to record, for each block of doc ids, the (term
- * freq, normalization factor) pairs that may trigger the maximum score of the block. This
- * information is recorded alongside skip data in order to be able to skip blocks of doc ids
- * if they may not produce high enough scores. Additionally doc values and norms has been
- * extended with jump-tables to make access O(1) instead of O(n), where n is the number of
- * elements to skip when advancing in the data.
- * <li>In version 8.4, postings, positions, offsets and payload lengths have move to a more
- * performant encoding that is vectorized.
- * <li>In version 8.6, index sort serialization is delegated to the sorts themselves, to allow
- * user-defined sorts to be used
- * <li>In version 8.7, stored fields compression became adaptive to better handle documents with
- * smaller stored fields.
- * <li>In version 9.0, vector-valued fields were added.
- * <li>In version 9.1, vector-valued fields were modified to add a graph hierarchy.
- * <li>In version 9.2, docs of vector-valued fields were moved from .vem to .vec and encoded by
- * IndexDISI. ordToDoc mappings was added to .vem.
- * <li>In version 9.5, HNSW graph connections were changed to be delta-encoded with vints.
- * Additionally, metadata file size improvements were made by delta-encoding nodes by graph
- * layer and not writing the node ids for the zeroth layer.
- * <li>In version 9.9, Vector scalar quantization support was added. Allowing the HNSW vector
- * format to utilize int8 quantized vectors for float32 vector search.
- * </ul>
- *
- * <a id="Limitations"></a>
- *
- * <h3>Limitations</h3>
- *
- * <div>
- *
- * <p>Lucene uses a Java <code>int</code> to refer to document numbers, and the index file format
- * uses an <code>Int32</code> on-disk to store document numbers. This is a limitation of both the
- * index file format and the current implementation. Eventually these should be replaced with either
- * <code>UInt64</code> values, or better yet, {@link org.apache.lucene.store.DataOutput#writeVInt
- * VInt} values which have no limit. </div>
- */
+/** Lucene 9.9 file format. */
package org.apache.lucene.codecs.lucene99;
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index c5a5ee36fd4..c0f08faaceb 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -21,8 +21,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
import org.apache.lucene.index.ImpactsEnum;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
@@ -398,10 +398,10 @@ public class PhraseQuery extends Query {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene99PostingsReader.BlockImpactsPostingsEnum#nextPosition()}.
+ * Lucene912PostingsReader.BlockImpactsPostingsEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene99PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene912PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
@@ -410,7 +410,7 @@ public class PhraseQuery extends Query {
/**
* Number of simple operations in {@link
- * Lucene99PostingsReader.BlockImpactsPostingsEnum#nextPosition()} when no seek or buffer refill
+ * Lucene912PostingsReader.BlockImpactsPostingsEnum#nextPosition()} when no seek or buffer refill
* is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
index 816097b3a66..8b672496601 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.Codec
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene99.Lucene99Codec
+org.apache.lucene.codecs.lucene912.Lucene912Codec
diff --git a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 0fd87daff7d..cd63926a287 100644
--- a/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat
+org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
index 86577c55fcd..2c00af51a94 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
@@ -18,7 +18,7 @@ package org.apache.lucene.codecs.lucene90;
import com.carrotsearch.randomizedtesting.generators.RandomPicks;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.DirectoryReader;
@@ -31,7 +31,7 @@ import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase;
public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFieldsFormatTestCase {
@Override
protected Codec getCodec() {
- return new Lucene99Codec(Lucene99Codec.Mode.BEST_COMPRESSION);
+ return new Lucene912Codec(Lucene912Codec.Mode.BEST_COMPRESSION);
}
/**
@@ -42,7 +42,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
for (int i = 0; i < 10; i++) {
IndexWriterConfig iwc = newIndexWriterConfig();
iwc.setCodec(
- new Lucene99Codec(RandomPicks.randomFrom(random(), Lucene99Codec.Mode.values())));
+ new Lucene912Codec(RandomPicks.randomFrom(random(), Lucene912Codec.Mode.values())));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig());
Document doc = new Document();
doc.add(new StoredField("field1", "value1"));
@@ -72,7 +72,7 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
expectThrows(
NullPointerException.class,
() -> {
- new Lucene99Codec(null);
+ new Lucene912Codec(null);
});
expectThrows(
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestLucene912PostingsFormat.java
similarity index 73%
copy from lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
copy to lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestLucene912PostingsFormat.java
index 7965bc9c778..1b8d0618c60 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/TestLucene90PostingsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene912/TestLucene912PostingsFormat.java
@@ -14,21 +14,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.lucene.backward_codecs.lucene90;
-
-import static org.apache.lucene.backward_codecs.lucene90.Lucene90ScoreSkipReader.readImpacts;
+package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.lucene.backward_codecs.lucene90.Lucene90ScoreSkipReader.MutableImpactList;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.CompetitiveImpactAccumulator;
import org.apache.lucene.codecs.lucene90.blocktree.FieldReader;
import org.apache.lucene.codecs.lucene90.blocktree.Stats;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99SkipWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader.MutableImpactList;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DirectoryReader;
@@ -36,6 +32,7 @@ import org.apache.lucene.index.Impact;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -44,12 +41,37 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
import org.apache.lucene.tests.util.TestUtil;
-public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
- private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene90RWPostingsFormat());
+public class TestLucene912PostingsFormat extends BasePostingsFormatTestCase {
@Override
protected Codec getCodec() {
- return codec;
+ return TestUtil.alwaysPostingsFormat(new Lucene912PostingsFormat());
+ }
+
+ public void testVInt15() throws IOException {
+ byte[] bytes = new byte[5];
+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ ByteArrayDataInput in = new ByteArrayDataInput();
+ for (int i : new int[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE}) {
+ out.reset(bytes);
+ Lucene912PostingsWriter.writeVInt15(out, i);
+ in.reset(bytes, 0, out.getPosition());
+ assertEquals(i, Lucene912PostingsReader.readVInt15(in));
+ assertEquals(out.getPosition(), in.getPosition());
+ }
+ }
+
+ public void testVLong15() throws IOException {
+ byte[] bytes = new byte[9];
+ ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ ByteArrayDataInput in = new ByteArrayDataInput();
+ for (long i : new long[] {0, 1, 127, 128, 32767, 32768, Integer.MAX_VALUE, Long.MAX_VALUE}) {
+ out.reset(bytes);
+ Lucene912PostingsWriter.writeVLong15(out, i);
+ in.reset(bytes, 0, out.getPosition());
+ assertEquals(i, Lucene912PostingsReader.readVLong15(in));
+ assertEquals(out.getPosition(), in.getPosition());
+ }
}
/** Make sure the final sub-block(s) are not skipped. */
@@ -77,22 +99,6 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
d.close();
}
- private void shouldFail(int minItemsInBlock, int maxItemsInBlock) {
- expectThrows(
- IllegalArgumentException.class,
- () -> {
- new Lucene99PostingsFormat(minItemsInBlock, maxItemsInBlock);
- });
- }
-
- public void testInvalidBlockSizes() throws Exception {
- shouldFail(0, 0);
- shouldFail(10, 8);
- shouldFail(-1, 10);
- shouldFail(10, -1);
- shouldFail(10, 12);
- }
-
public void testImpactSerialization() throws IOException {
// omit norms and omit freqs
doTestImpactSerialization(Collections.singletonList(new Impact(1, 1L)));
@@ -135,12 +141,15 @@ public class TestLucene90PostingsFormat extends BasePostingsFormatTestCase {
}
try (Directory dir = newDirectory()) {
try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
- Lucene99SkipWriter.writeImpacts(acc, out);
+ Lucene912PostingsWriter.writeImpacts(acc.getCompetitiveFreqNormPairs(), out);
}
try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
byte[] b = new byte[Math.toIntExact(in.length())];
in.readBytes(b, 0, b.length);
- List<Impact> impacts2 = readImpacts(new ByteArrayDataInput(b), new MutableImpactList());
+ List<Impact> impacts2 =
+ Lucene912PostingsReader.readImpacts(
+ new ByteArrayDataInput(b),
+ new MutableImpactList(impacts.size() + random().nextInt(3)));
assertEquals(impacts, impacts2);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
index 0bc8caeb3c6..544ff7448aa 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99HnswQuantizedVectorsFormat.java
@@ -30,6 +30,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -80,7 +81,7 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
@Override
protected Codec getCodec() {
- return new Lucene99Codec() {
+ return new Lucene912Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return format;
@@ -96,7 +97,7 @@ public class TestLucene99HnswQuantizedVectorsFormat extends BaseKnnVectorsFormat
dir,
newIndexWriterConfig()
.setCodec(
- new Lucene99Codec() {
+ new Lucene912Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new Lucene99HnswScalarQuantizedVectorsFormat(
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
index 8e6ba2ea300..3a182c2a22c 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorScorer.java
@@ -27,6 +27,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
import org.apache.lucene.codecs.hnsw.DefaultFlatVectorScorer;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -51,7 +52,7 @@ import org.apache.lucene.util.quantization.ScalarQuantizer;
public class TestLucene99ScalarQuantizedVectorScorer extends LuceneTestCase {
private static Codec getCodec(int bits, boolean compress) {
- return new Lucene99Codec() {
+ return new Lucene912Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return new Lucene99HnswScalarQuantizedVectorsFormat(
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
index c44dc0a3c8f..edf127490c7 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99ScalarQuantizedVectorsFormat.java
@@ -28,6 +28,7 @@ import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.FilterCodec;
import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.KnnVectorsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.KnnFloatVectorField;
@@ -67,7 +68,7 @@ public class TestLucene99ScalarQuantizedVectorsFormat extends BaseKnnVectorsForm
@Override
protected Codec getCodec() {
- return new Lucene99Codec() {
+ return new Lucene912Codec() {
@Override
public KnnVectorsFormat getKnnVectorsFormatForField(String field) {
return format;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
deleted file mode 100644
index 310d8524be9..00000000000
--- a/lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.index;
-
-import java.io.IOException;
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.store.ByteBuffersDirectory;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.FilterIndexInput;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.tests.analysis.MockAnalyzer;
-import org.apache.lucene.tests.analysis.MockTokenizer;
-import org.apache.lucene.tests.store.MockDirectoryWrapper;
-import org.apache.lucene.tests.util.LuceneTestCase;
-import org.apache.lucene.tests.util.TestUtil;
-import org.apache.lucene.util.BytesRef;
-
-/** Tests lazy skipping on the proximity file. */
-public class TestLazyProxSkipping extends LuceneTestCase {
- private IndexSearcher searcher;
- private int seeksCounter = 0;
-
- private String field = "tokens";
- private String term1 = "xx";
- private String term2 = "yy";
- private String term3 = "zz";
-
- private class SeekCountingDirectory extends MockDirectoryWrapper {
- public SeekCountingDirectory(Directory delegate) {
- super(random(), delegate);
- }
-
- @Override
- public IndexInput openInput(String name, IOContext context) throws IOException {
- IndexInput ii = super.openInput(name, context);
- if (name.endsWith(".prx") || name.endsWith(".pos")) {
- // we decorate the proxStream with a wrapper class that allows to count the number of calls
- // of seek()
- ii = new SeeksCountingStream(ii);
- }
- return ii;
- }
- }
-
- private void createIndex(int numHits) throws IOException {
- int numDocs = 500;
-
- final Analyzer analyzer =
- new Analyzer() {
- @Override
- public TokenStreamComponents createComponents(String fieldName) {
- return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
- }
- };
- Directory directory = new SeekCountingDirectory(new ByteBuffersDirectory());
- // note: test explicitly disables payloads
- IndexWriter writer =
- new IndexWriter(
- directory,
- newIndexWriterConfig(analyzer)
- .setMaxBufferedDocs(10)
- .setMergePolicy(newLogMergePolicy(false)));
-
- for (int i = 0; i < numDocs; i++) {
- Document doc = new Document();
- String content;
- if (i % (numDocs / numHits) == 0) {
- // add a document that matches the query "term1 term2"
- content = this.term1 + " " + this.term2;
- } else if (i % 15 == 0) {
- // add a document that only contains term1
- content = this.term1 + " " + this.term1;
- } else {
- // add a document that contains term2 but not term 1
- content = this.term3 + " " + this.term2;
- }
-
- doc.add(newTextField(this.field, content, Field.Store.YES));
- writer.addDocument(doc);
- }
-
- // make sure the index has only a single segment
- writer.forceMerge(1);
- writer.close();
-
- LeafReader reader = getOnlyLeafReader(DirectoryReader.open(directory));
-
- this.searcher = newSearcher(reader);
- }
-
- private ScoreDoc[] search() throws IOException {
- // create PhraseQuery "term1 term2" and search
- PhraseQuery pq = new PhraseQuery(field, term1, term2);
- return this.searcher.search(pq, 1000).scoreDocs;
- }
-
- private void performTest(int numHits) throws IOException {
- createIndex(numHits);
- this.seeksCounter = 0;
- ScoreDoc[] hits = search();
- // verify that the right number of docs was found
- assertEquals(numHits, hits.length);
-
- // check if the number of calls of seek() does not exceed the number of hits
- assertTrue(this.seeksCounter > 0);
- assertTrue(
- "seeksCounter=" + this.seeksCounter + " numHits=" + numHits,
- this.seeksCounter <= numHits + 1);
- searcher.getIndexReader().close();
- }
-
- public void testLazySkipping() throws IOException {
- final String fieldFormat = TestUtil.getPostingsFormat(this.field);
- assumeFalse("This test cannot run with Direct postings format", fieldFormat.equals("Direct"));
- assumeFalse(
- "This test cannot run with SimpleText postings format", fieldFormat.equals("SimpleText"));
-
- // test whether only the minimum amount of seeks()
- // are performed
- performTest(5);
- performTest(10);
- }
-
- public void testSeek() throws IOException {
- Directory directory = newDirectory();
- IndexWriter writer =
- new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random())));
- for (int i = 0; i < 10; i++) {
- Document doc = new Document();
- doc.add(newTextField(this.field, "a b", Field.Store.YES));
- writer.addDocument(doc);
- }
-
- writer.close();
- IndexReader reader = DirectoryReader.open(directory);
-
- PostingsEnum tp = MultiTerms.getTermPostingsEnum(reader, this.field, new BytesRef("b"));
-
- for (int i = 0; i < 10; i++) {
- tp.nextDoc();
- assertEquals(tp.docID(), i);
- assertEquals(tp.nextPosition(), 1);
- }
-
- tp = MultiTerms.getTermPostingsEnum(reader, this.field, new BytesRef("a"));
-
- for (int i = 0; i < 10; i++) {
- tp.nextDoc();
- assertEquals(tp.docID(), i);
- assertEquals(tp.nextPosition(), 0);
- }
- reader.close();
- directory.close();
- }
-
- // Simply extends IndexInput in a way that we are able to count the number
- // of invocations of seek()
- class SeeksCountingStream extends FilterIndexInput {
-
- SeeksCountingStream(IndexInput input) {
- super("SeekCountingStream(" + input + ")", input);
- }
-
- @Override
- public void seek(long pos) throws IOException {
- TestLazyProxSkipping.this.seeksCounter++;
- in.seek(pos);
- }
-
- @Override
- public SeeksCountingStream clone() {
- return new SeeksCountingStream(in.clone());
- }
- }
-}
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
index 2a5d8fffd01..e5ca29b9fae 100644
--- a/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/intervals/TermIntervalsSource.java
@@ -21,8 +21,8 @@ import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.Objects;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.PostingsEnum;
@@ -261,10 +261,10 @@ class TermIntervalsSource extends IntervalsSource {
/**
* A guess of the average number of simple operations for the initial seek and buffer refill per
* document for the positions of a term. See also {@link
- * Lucene99PostingsReader.EverythingEnum#nextPosition()}.
+ * Lucene912PostingsReader.EverythingEnum#nextPosition()}.
*
* <p>Aside: Instead of being constant this could depend among others on {@link
- * Lucene99PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
+ * Lucene912PostingsFormat#BLOCK_SIZE}, {@link TermsEnum#docFreq()}, {@link
* TermsEnum#totalTermFreq()}, {@link DocIdSetIterator#cost()} (expected number of matching docs),
* {@link LeafReader#maxDoc()} (total number of docs in the segment), and the seek time and block
* size of the device storing the index.
@@ -272,7 +272,7 @@ class TermIntervalsSource extends IntervalsSource {
private static final int TERM_POSNS_SEEK_OPS_PER_DOC = 128;
/**
- * Number of simple operations in {@link Lucene99PostingsReader.EverythingEnum#nextPosition()}
+ * Number of simple operations in {@link Lucene912PostingsReader.EverythingEnum#nextPosition()}
* when no seek or buffer refill is done.
*/
private static final int TERM_OPS_PER_POS = 7;
diff --git a/lucene/suggest/src/java/module-info.java b/lucene/suggest/src/java/module-info.java
index 673fa391213..c3f5b7c178f 100644
--- a/lucene/suggest/src/java/module-info.java
+++ b/lucene/suggest/src/java/module-info.java
@@ -31,7 +31,8 @@ module org.apache.lucene.suggest {
org.apache.lucene.search.suggest.document.Completion50PostingsFormat,
org.apache.lucene.search.suggest.document.Completion84PostingsFormat,
org.apache.lucene.search.suggest.document.Completion90PostingsFormat,
- org.apache.lucene.search.suggest.document.Completion99PostingsFormat;
+ org.apache.lucene.search.suggest.document.Completion99PostingsFormat,
+ org.apache.lucene.search.suggest.document.Completion912PostingsFormat;
provides org.apache.lucene.analysis.TokenFilterFactory with
org.apache.lucene.search.suggest.analyzing.SuggestStopFilterFactory;
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java
similarity index 69%
copy from lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
copy to lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java
index 4449d754e29..341b034e8e8 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion912PostingsFormat.java
@@ -20,26 +20,26 @@ import org.apache.lucene.codecs.PostingsFormat;
/**
* {@link CompletionPostingsFormat} for {@link
- * org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}
+ * org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat}
*
* @lucene.experimental
*/
-public class Completion99PostingsFormat extends CompletionPostingsFormat {
- /** Creates a {@link Completion99PostingsFormat} that will load the completion FST on-heap. */
- public Completion99PostingsFormat() {
+public class Completion912PostingsFormat extends CompletionPostingsFormat {
+ /** Creates a {@link Completion912PostingsFormat} that will load the completion FST on-heap. */
+ public Completion912PostingsFormat() {
this(FSTLoadMode.ON_HEAP);
}
/**
- * Creates a {@link Completion99PostingsFormat} that will use the provided <code>fstLoadMode
+ * Creates a {@link Completion912PostingsFormat} that will use the provided <code>fstLoadMode
* </code> to determine if the completion FST should be loaded on or off heap.
*/
- public Completion99PostingsFormat(FSTLoadMode fstLoadMode) {
- super("Completion99", fstLoadMode);
+ public Completion912PostingsFormat(FSTLoadMode fstLoadMode) {
+ super("Completion912", fstLoadMode);
}
@Override
protected PostingsFormat delegatePostingsFormat() {
- return PostingsFormat.forName("Lucene99");
+ return PostingsFormat.forName("Lucene912");
}
}
diff --git a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
index 4449d754e29..d3ea76d5425 100644
--- a/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
+++ b/lucene/suggest/src/java/org/apache/lucene/search/suggest/document/Completion99PostingsFormat.java
@@ -19,8 +19,10 @@ package org.apache.lucene.search.suggest.document;
import org.apache.lucene.codecs.PostingsFormat;
/**
- * {@link CompletionPostingsFormat} for {@link
- * org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat}
+ * {@link org.apache.lucene.search.suggest.document.CompletionPostingsFormat} for {@code
+ * org.apache.lucene.backward_codecs.lucene99.Lucene99PostingsFormat}. This format is only used for
+ * backward-compatibility of the index format and cannot be used to write data, use {@link
+ * Completion912PostingsFormat} on new indices.
*
* @lucene.experimental
*/
diff --git a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
index 81d1ed9465d..8544a97b88f 100644
--- a/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
+++ b/lucene/suggest/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
@@ -34,3 +34,4 @@ org.apache.lucene.search.suggest.document.Completion50PostingsFormat
org.apache.lucene.search.suggest.document.Completion84PostingsFormat
org.apache.lucene.search.suggest.document.Completion90PostingsFormat
org.apache.lucene.search.suggest.document.Completion99PostingsFormat
+org.apache.lucene.search.suggest.document.Completion912PostingsFormat
diff --git a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
index 30201a8eac8..a8b4b6a8931 100644
--- a/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
+++ b/lucene/suggest/src/test/org/apache/lucene/search/suggest/document/TestSuggestField.java
@@ -965,7 +965,7 @@ public class TestSuggestField extends LuceneTestCase {
new FilterCodec(TestUtil.getDefaultCodec().getName(), TestUtil.getDefaultCodec()) {
CompletionPostingsFormat.FSTLoadMode fstLoadMode =
RandomPicks.randomFrom(random(), CompletionPostingsFormat.FSTLoadMode.values());
- PostingsFormat postingsFormat = new Completion99PostingsFormat(fstLoadMode);
+ PostingsFormat postingsFormat = new Completion912PostingsFormat(fstLoadMode);
@Override
public PostingsFormat postingsFormat() {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
index 0a0c8f673db..9688c6c0e35 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneFixedGap.java
@@ -28,9 +28,9 @@ import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.FixedGapTermsIndexWriter;
import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -38,7 +38,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene99PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
+ * Customized version of {@link Lucene912PostingsFormat} that uses {@link FixedGapTermsIndexWriter}.
*/
public final class LuceneFixedGap extends PostingsFormat {
final int termIndexInterval;
@@ -54,7 +54,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene99PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene912PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -91,7 +91,7 @@ public final class LuceneFixedGap extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene99PostingsReader(state);
+ PostingsReaderBase postings = new Lucene912PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
index a864263b7c3..1fafc8f448c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapDocFreqInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene99PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene912PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval, but forcing high docfreq terms to be indexed
* terms.
*/
@@ -59,7 +59,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene99PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene912PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -100,7 +100,7 @@ public final class LuceneVarGapDocFreqInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene99PostingsReader(state);
+ PostingsReaderBase postings = new Lucene912PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
index 667acf0ceb3..bfaf0914651 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/blockterms/LuceneVarGapFixedInterval.java
@@ -29,9 +29,9 @@ import org.apache.lucene.codecs.blockterms.TermsIndexReaderBase;
import org.apache.lucene.codecs.blockterms.TermsIndexWriterBase;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexReader;
import org.apache.lucene.codecs.blockterms.VariableGapTermsIndexWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; // javadocs
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
@@ -39,7 +39,7 @@ import org.apache.lucene.index.SegmentWriteState;
// any PostingsFormat and make it ord-able...
/**
- * Customized version of {@link Lucene99PostingsFormat} that uses {@link
+ * Customized version of {@link Lucene912PostingsFormat} that uses {@link
* VariableGapTermsIndexWriter} with a fixed interval.
*/
public final class LuceneVarGapFixedInterval extends PostingsFormat {
@@ -56,7 +56,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
- PostingsWriterBase docs = new Lucene99PostingsWriter(state);
+ PostingsWriterBase docs = new Lucene912PostingsWriter(state);
// TODO: should we make the terms index more easily
// pluggable? Ie so that this codec would record which
@@ -95,7 +95,7 @@ public final class LuceneVarGapFixedInterval extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- PostingsReaderBase postings = new Lucene99PostingsReader(state);
+ PostingsReaderBase postings = new Lucene912PostingsReader(state);
TermsIndexReaderBase indexReader;
boolean success = false;
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
index dd8d6fedc77..7bd3bc9b034 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/mockrandom/MockRandomPostingsFormat.java
@@ -37,8 +37,8 @@ import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsReader;
import org.apache.lucene.codecs.blocktreeords.OrdsBlockTreeTermsWriter;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsReader;
import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.codecs.memory.FSTTermsReader;
import org.apache.lucene.codecs.memory.FSTTermsWriter;
import org.apache.lucene.index.FieldInfo;
@@ -121,7 +121,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
random.nextInt(); // consume a random for buffersize
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(state);
+ PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(state);
final FieldsConsumer fields;
final int t1 = random.nextInt(4);
@@ -289,7 +289,7 @@ public final class MockRandomPostingsFormat extends PostingsFormat {
System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize);
}
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(state);
+ PostingsReaderBase postingsReader = new Lucene912PostingsReader(state);
final FieldsProducer fields;
final int t1 = random.nextInt(4);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
index 0c469b392a2..4b13531327b 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/codecs/uniformsplit/UniformSplitRot13PostingsFormat.java
@@ -23,8 +23,8 @@ import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.PostingsReaderBase;
import org.apache.lucene.codecs.PostingsWriterBase;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsReader;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsWriter;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsReader;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsWriter;
import org.apache.lucene.codecs.uniformsplit.BlockDecoder;
import org.apache.lucene.codecs.uniformsplit.BlockEncoder;
import org.apache.lucene.codecs.uniformsplit.IndexDictionary;
@@ -67,7 +67,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState segmentWriteState) throws IOException {
- PostingsWriterBase postingsWriter = new Lucene99PostingsWriter(segmentWriteState);
+ PostingsWriterBase postingsWriter = new Lucene912PostingsWriter(segmentWriteState);
boolean success = false;
try {
FieldsConsumer fieldsConsumer = createFieldsConsumer(segmentWriteState, postingsWriter);
@@ -145,7 +145,7 @@ public class UniformSplitRot13PostingsFormat extends PostingsFormat {
@Override
public FieldsProducer fieldsProducer(SegmentReadState segmentReadState) throws IOException {
- PostingsReaderBase postingsReader = new Lucene99PostingsReader(segmentReadState);
+ PostingsReaderBase postingsReader = new Lucene912PostingsReader(segmentReadState);
boolean success = false;
try {
FieldsProducer fieldsProducer = createFieldsProducer(segmentReadState, postingsReader);
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
index c386a7ef85e..378444e394a 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestRuleSetupAndRestoreClassEnv.java
@@ -38,7 +38,7 @@ import java.util.TimeZone;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
import org.apache.lucene.codecs.simpletext.SimpleTextCodec;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
@@ -188,9 +188,9 @@ final class TestRuleSetupAndRestoreClassEnv extends AbstractBeforeAfterRule {
} else if ("Compressing".equals(TEST_CODEC)
|| ("random".equals(TEST_CODEC) && randomVal == 6 && !shouldAvoidCodec("Compressing"))) {
codec = CompressingCodec.randomInstance(random);
- } else if ("Lucene99".equals(TEST_CODEC)
- || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene99"))) {
- codec = new Lucene99Codec(RandomPicks.randomFrom(random, Lucene99Codec.Mode.values()));
+ } else if ("Lucene912".equals(TEST_CODEC)
+ || ("random".equals(TEST_CODEC) && randomVal == 5 && !shouldAvoidCodec("Lucene912"))) {
+ codec = new Lucene912Codec(RandomPicks.randomFrom(random, Lucene912Codec.Mode.values()));
} else if (!"random".equals(TEST_CODEC)) {
codec = Codec.forName(TEST_CODEC);
} else if ("random".equals(TEST_POSTINGSFORMAT)) {
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
index 492eef8ed79..159846c9bca 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestUtil.java
@@ -54,9 +54,9 @@ import org.apache.lucene.codecs.KnnVectorsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.blocktreeords.BlockTreeOrdsPostingsFormat;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912Codec;
+import org.apache.lucene.codecs.lucene912.Lucene912PostingsFormat;
import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat;
-import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat;
import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat;
import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat;
import org.apache.lucene.document.BinaryDocValuesField;
@@ -1238,7 +1238,7 @@ public final class TestUtil {
* different than {@link Codec#getDefault()} because that is randomized.
*/
public static Codec getDefaultCodec() {
- return new Lucene99Codec();
+ return new Lucene912Codec();
}
/**
@@ -1246,7 +1246,7 @@ public final class TestUtil {
* Lucene.
*/
public static PostingsFormat getDefaultPostingsFormat() {
- return new Lucene99PostingsFormat();
+ return new Lucene912PostingsFormat();
}
/**
@@ -1257,7 +1257,7 @@ public final class TestUtil {
*/
public static PostingsFormat getDefaultPostingsFormat(
int minItemsPerBlock, int maxItemsPerBlock) {
- return new Lucene99PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
+ return new Lucene912PostingsFormat(minItemsPerBlock, maxItemsPerBlock);
}
/** Returns a random postings format that supports term ordinals */
(lucene) 01/02: Bump the window size of disjunction from 2,048 to 4,096. (#13605)
Posted by jp...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 24de36cc28ce884d31d6e9a730f96aff1b7e62bf
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Thu Jul 25 15:38:21 2024 +0200
Bump the window size of disjunction from 2,048 to 4,096. (#13605)
It's been pointed multiple times that a difference between Tantivy and Lucene
is the fact that Tantivy uses windows of 4,096 docs when Lucene has a 2x
smaller window size of 2,048 docs and that this might explain part of the
performance difference. luceneutil suggests that bumping the window size to
4,096 does indeed improve performance for counting queries, but not for top-k
queries. I'm still suggesting to bump the window size across the board to keep
our disjunction scorer consistent.
---
.../org/apache/lucene/search/BooleanScorer.java | 2 +-
.../apache/lucene/search/MaxScoreBulkScorer.java | 2 +-
.../lucene/search/TestMaxScoreBulkScorer.java | 26 +++++++++++-----------
3 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
index 9d791d3c33d..f148cb7eb00 100644
--- a/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
@@ -30,7 +30,7 @@ import org.apache.lucene.util.PriorityQueue;
*/
final class BooleanScorer extends BulkScorer {
- static final int SHIFT = 11;
+ static final int SHIFT = 12;
static final int SIZE = 1 << SHIFT;
static final int MASK = SIZE - 1;
static final int SET_SIZE = 1 << (SHIFT - 6);
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
index ef8627b7ab4..6c47f68302e 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreBulkScorer.java
@@ -25,7 +25,7 @@ import org.apache.lucene.util.MathUtil;
final class MaxScoreBulkScorer extends BulkScorer {
- static final int INNER_WINDOW_SIZE = 1 << 11;
+ static final int INNER_WINDOW_SIZE = 1 << 12;
private final int maxDoc;
// All scorers, sorted by increasing max score.
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java
index c000c82eb13..eb6281ade9a 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreBulkScorer.java
@@ -125,19 +125,19 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
assertEquals(2 + 1, scorer.score(), 0);
break;
case 1:
- assertEquals(2048, doc);
+ assertEquals(4096, doc);
assertEquals(2, scorer.score(), 0);
break;
case 2:
- assertEquals(6144, doc);
+ assertEquals(12288, doc);
assertEquals(2 + 1, scorer.score(), 0);
break;
case 3:
- assertEquals(8192, doc);
+ assertEquals(16384, doc);
assertEquals(1, scorer.score(), 0);
break;
case 4:
- assertEquals(10240, doc);
+ assertEquals(20480, doc);
assertEquals(1, scorer.score(), 0);
break;
default:
@@ -195,13 +195,13 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
assertEquals(2 + 1, scorer.score(), 0);
break;
case 1:
- assertEquals(2048, doc);
+ assertEquals(4096, doc);
assertEquals(2, scorer.score(), 0);
// simulate top-2 retrieval
scorer.setMinCompetitiveScore(Math.nextUp(2));
break;
case 2:
- assertEquals(6144, doc);
+ assertEquals(12288, doc);
assertEquals(2 + 1, scorer.score(), 0);
scorer.setMinCompetitiveScore(Math.nextUp(2 + 1));
break;
@@ -268,19 +268,19 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
assertEquals(2 + 1, scorer.score(), 0);
break;
case 1:
- assertEquals(2048, doc);
+ assertEquals(4096, doc);
assertEquals(2, scorer.score(), 0);
break;
case 2:
- assertEquals(6144, doc);
+ assertEquals(12288, doc);
assertEquals(2 + 1 + 3, scorer.score(), 0);
break;
case 3:
- assertEquals(8192, doc);
+ assertEquals(16384, doc);
assertEquals(1, scorer.score(), 0);
break;
case 4:
- assertEquals(10240, doc);
+ assertEquals(20480, doc);
assertEquals(1 + 3, scorer.score(), 0);
break;
default:
@@ -346,18 +346,18 @@ public class TestMaxScoreBulkScorer extends LuceneTestCase {
assertEquals(2 + 1, scorer.score(), 0);
break;
case 1:
- assertEquals(2048, doc);
+ assertEquals(4096, doc);
assertEquals(2, scorer.score(), 0);
// simulate top-2 retrieval
scorer.setMinCompetitiveScore(Math.nextUp(2));
break;
case 2:
- assertEquals(6144, doc);
+ assertEquals(12288, doc);
assertEquals(2 + 1 + 3, scorer.score(), 0);
scorer.setMinCompetitiveScore(Math.nextUp(2 + 1));
break;
case 3:
- assertEquals(10240, doc);
+ assertEquals(20480, doc);
assertEquals(1 + 3, scorer.score(), 0);
scorer.setMinCompetitiveScore(Math.nextUp(1 + 3));
break;