You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/12/11 17:39:24 UTC

(lucene) branch branch_9_9 updated: Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)

This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a commit to branch branch_9_9
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9_9 by this push:
     new 69457d8a113 Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)
69457d8a113 is described below

commit 69457d8a113519ad2dca9d9c01c17df584e190c6
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Mon Dec 11 18:14:28 2023 +0100

    Add tests for the 9.0->9.8 block tree terms dict format back. (#12908)
    
    I just noticed that the move from FOR to PFOR did all the work to make the old
    format (FOR) writeable, but missed keeping an instance of
    `BasePostingsFormatTestCase` for this format.
---
 .../lucene90/Lucene90RWPostingsFormat.java         |  6 ++-
 .../blocktree/Lucene90BlockTreeTermsWriter.java    | 53 +++++++++++++++++++---
 .../src/java/org/apache/lucene/util/fst/FST.java   | 19 ++++++--
 .../org/apache/lucene/util/fst/FSTCompiler.java    | 29 ++++++++++--
 4 files changed, 92 insertions(+), 15 deletions(-)

diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
index 6497bb13d11..4360b90f237 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene90/Lucene90RWPostingsFormat.java
@@ -75,7 +75,11 @@ public final class Lucene90RWPostingsFormat extends PostingsFormat {
     try {
       FieldsConsumer ret =
           new Lucene90BlockTreeTermsWriter(
-              state, postingsWriter, minTermBlockSize, maxTermBlockSize);
+              state,
+              postingsWriter,
+              minTermBlockSize,
+              maxTermBlockSize,
+              Lucene90BlockTreeTermsReader.VERSION_START);
       success = true;
       return ret;
     } finally {
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
index 0dded44d6c4..cd204de167e 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/Lucene90BlockTreeTermsWriter.java
@@ -236,6 +236,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
   final int maxDoc;
   final int minItemsInBlock;
   final int maxItemsInBlock;
+  final int version;
 
   final PostingsWriterBase postingsWriter;
   final FieldInfos fieldInfos;
@@ -253,10 +254,37 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
       int minItemsInBlock,
       int maxItemsInBlock)
       throws IOException {
+    this(
+        state,
+        postingsWriter,
+        minItemsInBlock,
+        maxItemsInBlock,
+        Lucene90BlockTreeTermsReader.VERSION_CURRENT);
+  }
+
+  /** Expert constructor that allows configuring the version, used for bw tests. */
+  public Lucene90BlockTreeTermsWriter(
+      SegmentWriteState state,
+      PostingsWriterBase postingsWriter,
+      int minItemsInBlock,
+      int maxItemsInBlock,
+      int version)
+      throws IOException {
     validateSettings(minItemsInBlock, maxItemsInBlock);
 
     this.minItemsInBlock = minItemsInBlock;
     this.maxItemsInBlock = maxItemsInBlock;
+    if (version < Lucene90BlockTreeTermsReader.VERSION_START
+        || version > Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+      throw new IllegalArgumentException(
+          "Expected version in range ["
+              + Lucene90BlockTreeTermsReader.VERSION_START
+              + ", "
+              + Lucene90BlockTreeTermsReader.VERSION_CURRENT
+              + "], but got "
+              + version);
+    }
+    this.version = version;
 
     this.maxDoc = state.segmentInfo.maxDoc();
     this.fieldInfos = state.fieldInfos;
@@ -274,7 +302,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
       CodecUtil.writeIndexHeader(
           termsOut,
           Lucene90BlockTreeTermsReader.TERMS_CODEC_NAME,
-          Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+          version,
           state.segmentInfo.getId(),
           state.segmentSuffix);
 
@@ -287,7 +315,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
       CodecUtil.writeIndexHeader(
           indexOut,
           Lucene90BlockTreeTermsReader.TERMS_INDEX_CODEC_NAME,
-          Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+          version,
           state.segmentInfo.getId(),
           state.segmentSuffix);
       // segment = state.segmentInfo.name;
@@ -301,7 +329,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
       CodecUtil.writeIndexHeader(
           metaOut,
           Lucene90BlockTreeTermsReader.TERMS_META_CODEC_NAME,
-          Lucene90BlockTreeTermsReader.VERSION_CURRENT,
+          version,
           state.segmentInfo.getId(),
           state.segmentSuffix);
 
@@ -449,7 +477,7 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
     scratchBytes.writeByte((byte) (((l >>> 57) & 0x7FL)));
   }
 
-  private static final class PendingBlock extends PendingEntry {
+  private final class PendingBlock extends PendingEntry {
     public final BytesRef prefix;
     public final long fp;
     public FST<BytesRef> index;
@@ -492,7 +520,11 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
       assert scratchBytes.size() == 0;
 
       // write the leading vLong in MSB order for better outputs sharing in the FST
-      writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
+      if (version >= Lucene90BlockTreeTermsReader.VERSION_MSB_VLONG_OUTPUT) {
+        writeMSBVLong(encodeOutput(fp, hasTerms, isFloor), scratchBytes);
+      } else {
+        scratchBytes.writeVLong(encodeOutput(fp, hasTerms, isFloor));
+      }
       if (isFloor) {
         scratchBytes.writeVInt(blocks.size() - 1);
         for (int i = 1; i < blocks.size(); i++) {
@@ -520,8 +552,17 @@ public final class Lucene90BlockTreeTermsWriter extends FieldsConsumer {
       int pageBits = Math.min(15, Math.max(6, estimateBitsRequired));
 
       final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton();
+      final int fstVersion;
+      if (version >= Lucene90BlockTreeTermsReader.VERSION_CURRENT) {
+        fstVersion = FST.VERSION_CURRENT;
+      } else {
+        fstVersion = FST.VERSION_90;
+      }
       final FSTCompiler<BytesRef> fstCompiler =
-          new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs).bytesPageBits(pageBits).build();
+          new FSTCompiler.Builder<>(FST.INPUT_TYPE.BYTE1, outputs)
+              .bytesPageBits(pageBits)
+              .setVersion(fstVersion)
+              .build();
       // if (DEBUG) {
       //  System.out.println("  compile index for prefix=" + prefix);
       // }
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index 0dd35618e8e..62beb6a6b11 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -107,10 +107,23 @@ public final class FST<T> implements Accountable {
 
   // Increment version to change it
   private static final String FILE_FORMAT_NAME = "FST";
-  private static final int VERSION_START = 6;
+
+  /** First supported version, this is the version that was used when releasing Lucene 7.0. */
+  public static final int VERSION_START = 6;
+
+  // Version 7 introduced direct addressing for arcs, but it's not recorded here because it doesn't
+  // need version checks on the read side, it uses new flag values on arcs instead.
+
   private static final int VERSION_LITTLE_ENDIAN = 8;
-  private static final int VERSION_CONTINUOUS_ARCS = 9;
-  static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;
+
+  /** Version that started storing continuous arcs. */
+  public static final int VERSION_CONTINUOUS_ARCS = 9;
+
+  /** Current version. */
+  public static final int VERSION_CURRENT = VERSION_CONTINUOUS_ARCS;
+
+  /** Version that was used when releasing Lucene 9.0. */
+  public static final int VERSION_90 = VERSION_LITTLE_ENDIAN;
 
   // Never serialized; just used to represent the virtual
   // final node w/ no arcs:
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
index be6bb663ca9..10de98911f9 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FSTCompiler.java
@@ -27,7 +27,6 @@ import static org.apache.lucene.util.fst.FST.BIT_STOP_NODE;
 import static org.apache.lucene.util.fst.FST.BIT_TARGET_NEXT;
 import static org.apache.lucene.util.fst.FST.FINAL_END_NODE;
 import static org.apache.lucene.util.fst.FST.NON_FINAL_END_NODE;
-import static org.apache.lucene.util.fst.FST.VERSION_CURRENT;
 import static org.apache.lucene.util.fst.FST.getNumPresenceBytes;
 
 import java.io.IOException;
@@ -118,6 +117,7 @@ public class FSTCompiler<T> {
 
   final boolean allowFixedLengthArcs;
   final float directAddressingMaxOversizingFactor;
+  final int version;
   long directAddressingExpansionCredit;
 
   final BytesStore bytes;
@@ -128,14 +128,16 @@ public class FSTCompiler<T> {
       Outputs<T> outputs,
       boolean allowFixedLengthArcs,
       int bytesPageBits,
-      float directAddressingMaxOversizingFactor) {
+      float directAddressingMaxOversizingFactor,
+      int version) {
     this.allowFixedLengthArcs = allowFixedLengthArcs;
     this.directAddressingMaxOversizingFactor = directAddressingMaxOversizingFactor;
+    this.version = version;
     bytes = new BytesStore(bytesPageBits);
     // pad: ensure no node gets address 0 which is reserved to mean
     // the stop state w/ no arcs
     bytes.writeByte((byte) 0);
-    fst = new FST<>(new FST.FSTMetadata<>(inputType, outputs, null, -1, VERSION_CURRENT, 0), bytes);
+    fst = new FST<>(new FST.FSTMetadata<>(inputType, outputs, null, -1, version, 0), bytes);
     if (suffixRAMLimitMB < 0) {
       throw new IllegalArgumentException("ramLimitMB must be >= 0; got: " + suffixRAMLimitMB);
     } else if (suffixRAMLimitMB > 0) {
@@ -167,6 +169,7 @@ public class FSTCompiler<T> {
     private boolean allowFixedLengthArcs = true;
     private int bytesPageBits = 15;
     private float directAddressingMaxOversizingFactor = DIRECT_ADDRESSING_MAX_OVERSIZING_FACTOR;
+    private int version = FST.VERSION_CURRENT;
 
     /**
      * @param inputType The input type (transition labels). Can be anything from {@link INPUT_TYPE}
@@ -246,6 +249,21 @@ public class FSTCompiler<T> {
       return this;
     }
 
+    /** Expert: Set the codec version. * */
+    public Builder<T> setVersion(int version) {
+      if (version < FST.VERSION_90 || version > FST.VERSION_CURRENT) {
+        throw new IllegalArgumentException(
+            "Expected version in range ["
+                + FST.VERSION_90
+                + ", "
+                + FST.VERSION_CURRENT
+                + "], got "
+                + version);
+      }
+      this.version = version;
+      return this;
+    }
+
     /** Creates a new {@link FSTCompiler}. */
     public FSTCompiler<T> build() throws IOException {
       return new FSTCompiler<>(
@@ -254,7 +272,8 @@ public class FSTCompiler<T> {
           outputs,
           allowFixedLengthArcs,
           bytesPageBits,
-          directAddressingMaxOversizingFactor);
+          directAddressingMaxOversizingFactor,
+          version);
     }
   }
 
@@ -437,7 +456,7 @@ public class FSTCompiler<T> {
       int labelRange = nodeIn.arcs[nodeIn.numArcs - 1].label - nodeIn.arcs[0].label + 1;
       assert labelRange > 0;
       boolean continuousLabel = labelRange == nodeIn.numArcs;
-      if (continuousLabel) {
+      if (continuousLabel && version >= FST.VERSION_CONTINUOUS_ARCS) {
         writeNodeForDirectAddressingOrContinuous(
             nodeIn, startAddress, maxBytesPerArcWithoutLabel, labelRange, true);
         continuousNodeCount++;