You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/12/11 17:39:24 UTC
(lucene) branch branch_9_9 updated: Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9_9
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9_9 by this push:
new 69457d8a113 Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)
69457d8a113 is described below
commit 69457d8a113519ad2dca9d9c01c17df584e190c6
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Mon Dec 11 18:14:28 2023 +0100
Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)
I just noticed that the move from FOR to PFOR did all the work to make the old
format (FOR) writeable, but missed keeping an instance of
`BasePostingsFormatTestCase` for this format.
---
.../lucene90/Lucene90RWPostingsFormat.java | 6 ++-
.../blocktree/Lucene90BlockTreeTermsWriter.java | 53 +++++++++++++++++++---
.../src/java/org/apache/lucene/util/fst/FST.java | 19 ++++++--
.../org/apache/lucene/util/fst/FSTCompiler.java | 29 ++++++++++--
4 files changed, 92 insertions(+), 15 deletions(-)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
index 6497bb13d11..4360b90f237 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
@@ -75,7 +75,11 @@ public final class Lucene90RWPostingsFormat extends PostingsFormat {
try {
FieldsConsumer ret =
new Lucene90BlockTreeTermsWriter(
- state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+ state,
+ postingsWriter,
+ minTermBlockSize,
+ maxTermBlockSize,
+ Lucene90BlockTreeTermsReader.VERSION_START);
success = true;
return ret;
} finally {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 0dded44d6c4..cd204de167e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -236,6 +236,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
final int maxDoc;
final int minItemsInBlock;
final int maxItemsInBlock;
+ final int version;
final PostingsWriterBase postingsWriter;
final FieldInfos fieldInfos;
@@ -253,10 +254,37 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
int minItemsInBlock,
int maxItemsInBlock)
throws IOException {
+ this(
+ state,
+ postingsWriter,
+ minItemsInBlock,
+ maxItemsInBlock,
+ Lucene90BlockTreeTermsReader.VERSION_CURRENT);
+ }
+
+ /** Expert constructor that allows configuring the version, used for bw tests. */
+ public Lucene90BlockTreeTermsWriter(
+ SegmentWriteState state,
+ PostingsWriterBase postingsWriter,
+ int minItemsInBlock,
+ int maxItemsInBlock,
+ int version)
+ throws IOException {
validateSettings(minItemsInBlock, maxItemsInBlock);
this.minItemsInBlock = minItemsInBlock;
this.maxItemsInBlock = maxItemsInBlock;
+ if (version < Lucene90BlockTreeTermsReader.VERSION_START
+ || version > Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+ throw new IllegalArgumentException(
+ "Expected version in range ["
+ + Lucene90BlockTreeTermsReader.VERSION_START
+ + ", "
+ + Lucene90BlockTreeTermsReader.VERSION_CURRENT
+ + "], but got "
+ + version);
+ }
+ this.version = version;
this.maxDoc = state.segmentInfo.maxDoc();
this.fieldInfos = state.fieldInfos;
@@ -274,7 +302,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
CodecUtil.writeIndexHeader(
termsOut,
Lucene90BlockTreeTermsReader.TERMS_CODEC_NAME,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+ version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -287,7 +315,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
CodecUtil.writeIndexHeader(
indexOut,
Lucene90BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+ version,
state.segmentInfo.getId(),
state.segmentSuffix);
// segment = state.segmentInfo.name;
@@ -301,7 +329,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
CodecUtil.writeIndexHeader(
metaOut,
Lucene90BlockTreeTermsReader.TERMS_META_CODEC_NAME,
- Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+ version,
state.segmentInfo.getId(),
state.segmentSuffix);
@@ -449,7 +477,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL)));
}
- private static final class PendingBlock extends PendingEntry {
+ private final class PendingBlock extends PendingEntry {
public final BytesRef prefix;
public final long fp;
public FST<BytesRef> index;
@@ -492,7 +520,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
assert scratchBytes.size() == 0;
// write the leading vLong in MSB order for better outputs sharing in the FST
- writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
+ if (version >= Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT) {
+ writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
+ } else {
+ scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
+ }
if (isFloor) {
scratchBytes.writeVInt(blocks.size() - 1);
for (int i = 1; i < blocks.size(); i++) {
@@ -520,8 +552,17 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));
final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+ final int fstVersion;
+ if (version >= Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+ fstVersion = FST.VERSION_CURRENT;
+ } else {
+ fstVersion = FST.VERSION_90;
+ }
final FSTCompiler<BytesRef> fstCompiler =
- new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).bytesPageBits(pageBits).build();
+ new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
+ .bytesPageBits(pageBits)
+ .setVersion(fstVersion)
+ .build();
// if (DEBUG) {
// System.out.println(" compile index for prefix=" + prefix);
// }
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index 0dd35618e8e..62beb6a6b11 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -107,10 +107,23 @@ public final class FST<T> implements Accountable {
// Increment version to change it
private static final String FILE_FORMAT_NAME = "FST";
- private static final int VERSION_START = 6;
+
+ /** First supported version, this is the version that was used when releasing Lucene 7.0. */
+ public static final int VERSION_START = 6;
+
+ // Version 7 introduced direct addressing for arcs, but it's not recorded here because it doesn't
+ // need version checks on the read side, it uses new flag values on arcs instead.
+
private static final int VERSION_LITTLE_ENDIAN = 8;
- private static final int VERSION_CONTINUOUS_ARCS = 9;
- static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;
+
+ /** Version that started storing continuous arcs. */
+ public static final int VERSION_CONTINUOUS_ARCS = 9;
+
+ /** Current version. */
+ public static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;
+
+ /** Version that was used when releasing Lucene 9.0. */
+ public static final int VERSION_90 = VERSION_LITTLE_ENDIAN;
// Never serialized; just used to represent the virtual
// final node w/ no arcs:
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
index be6bb663ca9..10de98911f9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -27,7 +27,6 @@ import static org.apache.lucene.util.fst.FST.BIT_STOP_NODE;
import static org.apache.lucene.util.fst.FST.BIT_TARGET_NEXT;
import static org.apache.lucene.util.fst.FST.FINAL_END_NODE;
import static org.apache.lucene.util.fst.FST.NON_FINAL_END_NODE;
-import static org.apache.lucene.util.fst.FST.VERSION_CURRENT;
import static org.apache.lucene.util.fst.FST.getNumPresenceBytes;
import java.io.IOException;
@@ -118,6 +117,7 @@ public class FSTCompiler<T> {
final boolean allowFixedLengthArcs;
final float directAddressingMaxOversizingFactor;
+ final int version;
long directAddressingExpansionCredit;
final BytesStore bytes;
@@ -128,14 +128,16 @@ public class FSTCompiler<T> {
Outputs<T> outputs,
boolean allowFixedLengthArcs,
int bytesPageBits,
- float directAddressingMaxOversizingFactor) {
+ float directAddressingMaxOversizingFactor,
+ int version) {
this.allowFixedLengthArcs = allowFixedLengthArcs;
this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor;
+ this.version = version;
bytes = new BytesStore(bytesPageBits);
// pad: ensure no node gets address 0 which is reserved to mean
// the stop state w/ no arcs
bytes.writeByte((byte) 0);
- fst = new FST<>(new FST.FSTMetadata<>(inputType, outputs, null, -1, VERSION_CURRENT, 0), bytes);
+ fst = new FST<>(new FST.FSTMetadata<>(inputType, outputs, null, -1, version, 0), bytes);
if (suffixRAMLimitMB < 0) {
throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
} else if (suffixRAMLimitMB > 0) {
@@ -167,6 +169,7 @@ public class FSTCompiler<T> {
private boolean allowFixedLengthArcs = true;
private int bytesPageBits = 15;
private float directAddressingMaxOversizingFactor = DIRECT_ADDRESSING_MAX_OVERSIZING_FACTOR;
+ private int version = FST.VERSION_CURRENT;
/**
* @param inputType The input type (transition labels). Can be anything from {@link INPUT_TYPE}
@@ -246,6 +249,21 @@ public class FSTCompiler<T> {
return this;
}
+ /** Expert: Set the codec version. * */
+ public Builder<T> setVersion(int version) {
+ if (version < FST.VERSION_90 || version > FST.VERSION_CURRENT) {
+ throw new IllegalArgumentException(
+ "Expected version in range ["
+ + FST.VERSION_90
+ + ", "
+ + FST.VERSION_CURRENT
+ + "], got "
+ + version);
+ }
+ this.version = version;
+ return this;
+ }
+
/** Creates a new {@link FSTCompiler}. */
public FSTCompiler<T> build() throws IOException {
return new FSTCompiler<>(
@@ -254,7 +272,8 @@ public class FSTCompiler<T> {
outputs,
allowFixedLengthArcs,
bytesPageBits,
- directAddressingMaxOversizingFactor);
+ directAddressingMaxOversizingFactor,
+ version);
}
}
@@ -437,7 +456,7 @@ public class FSTCompiler<T> {
int labelRange = nodeIn.arcs[nodeIn.numArcs - 1].label - nodeIn.arcs[0].label + 1;
assert labelRange > 0;
boolean continuousLabel = labelRange == nodeIn.numArcs;
- if (continuousLabel) {
+ if (continuousLabel && version >= FST.VERSION_CONTINUOUS_ARCS) {
writeNodeForDirectAddressingOrContinuous(
nodeIn, startAddress, maxBytesPerArcWithoutLabel, labelRange, true);
continuousNodeCount++;