You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2023/12/14 12:38:41 UTC

(lucene) branch main updated: Ensure Nori/Kuromoji shipped binary FST is the latest version (#12933)

This is an automated email from the ASF dual-hosted git repository.

mikemccand pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/main by this push:
     new ebf9e295706 Ensure Nori/Kuromoji shipped binary FST is the latest version (#12933)
ebf9e295706 is described below

commit ebf9e29570628d55302922f89686e67fca06c620
Author: Michael McCandless <mi...@apache.org>
AuthorDate: Thu Dec 14 07:38:34 2023 -0500

    Ensure Nori/Kuromoji shipped binary FST is the latest version (#12933)
    
    * ensure Nori/Kuromoji shipped binary FST is the latest version (closes #12911)
    
    * fold feedback from @uschindler: sharpen test failure methods to give the specific gradlew command to regenerate the precise FST (not everything)
    
    * add javadoc for FSTMetadata.getVersion
---
 .../analysis/ja/dict/TokenInfoDictionary.java      |  2 +-
 .../analysis/ja/dict/TestTokenInfoDictionary.java  | 25 ++++++++++++++++++++++
 .../analysis/ko/dict/TokenInfoDictionary.java      |  2 +-
 .../analysis/ko/dict/TestTokenInfoDictionary.java  | 25 ++++++++++++++++++++++
 .../src/java/org/apache/lucene/util/fst/FST.java   | 11 +++++++++-
 5 files changed, 62 insertions(+), 3 deletions(-)

diff --git a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
index 7fd5a9244c1..ac2520f5b62 100644
--- a/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
+++ b/lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/TokenInfoDictionary.java
@@ -111,7 +111,7 @@ public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphDa
     this.fst = new TokenInfoFST(fst, true);
   }
 
-  private static InputStream getClassResource(String suffix) throws IOException {
+  static InputStream getClassResource(String suffix) throws IOException {
     final String resourcePath = TokenInfoDictionary.class.getSimpleName() + suffix;
     return IOUtils.requireResourceNonNull(
         TokenInfoDictionary.class.getResourceAsStream(resourcePath), resourcePath);
diff --git a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
index d365ca096c8..d76a6dcfd58 100644
--- a/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
+++ b/lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/dict/TestTokenInfoDictionary.java
@@ -21,18 +21,22 @@ import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SU
 import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
 import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
 
+import java.io.BufferedInputStream;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import org.apache.lucene.store.InputStreamDataInput;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.IntsRefBuilder;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.IntsRefFSTEnum;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
 
 /** Tests of TokenInfoDictionary build tools; run using ant test-tools */
 public class TestTokenInfoDictionary extends LuceneTestCase {
@@ -178,4 +182,25 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
       System.out.println("checked " + numTerms + " terms, " + numWords + " words.");
     }
   }
+
+  // #12911: make sure our shipped binary FST for TokenInfoDictionary is the latest & greatest
+  // format
+  public void testBinaryFSTIsLatestFormat() throws Exception {
+    try (InputStream is =
+        new BufferedInputStream(
+            TokenInfoDictionary.getClassResource(TokenInfoDictionary.FST_FILENAME_SUFFIX))) {
+      // we only need to load the FSTMetadata to check version:
+      int actualVersion =
+          FST.readMetadata(new InputStreamDataInput(is), PositiveIntOutputs.getSingleton())
+              .getVersion();
+      assertEquals(
+          "TokenInfoDictionary's FST is not the latest version: expected "
+              + FST.VERSION_CURRENT
+              + " but got: "
+              + actualVersion
+              + "; run \"./gradlew :lucene:analysis:kuromoji:regenerate\" to regenerate this FST",
+          FST.VERSION_CURRENT,
+          actualVersion);
+    }
+  }
 }
diff --git a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
index 07540075da0..679fb873988 100644
--- a/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
+++ b/lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/TokenInfoDictionary.java
@@ -109,7 +109,7 @@ public final class TokenInfoDictionary extends BinaryDictionary<TokenInfoMorphDa
     this.fst = new TokenInfoFST(fst);
   }
 
-  private static InputStream getClassResource(String suffix) throws IOException {
+  static InputStream getClassResource(String suffix) throws IOException {
     final String resourcePath = TokenInfoDictionary.class.getSimpleName() + suffix;
     return IOUtils.requireResourceNonNull(
         TokenInfoDictionary.class.getResourceAsStream(resourcePath), resourcePath);
diff --git a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
index a0d8f9b778f..7c6b282eecf 100644
--- a/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
+++ b/lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/dict/TestTokenInfoDictionary.java
@@ -21,6 +21,8 @@ import static org.apache.lucene.analysis.morph.BinaryDictionary.DICT_FILENAME_SU
 import static org.apache.lucene.analysis.morph.BinaryDictionary.POSDICT_FILENAME_SUFFIX;
 import static org.apache.lucene.analysis.morph.BinaryDictionary.TARGETMAP_FILENAME_SUFFIX;
 
+import java.io.BufferedInputStream;
+import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
@@ -28,12 +30,14 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import org.apache.lucene.analysis.ko.POS;
+import org.apache.lucene.store.InputStreamDataInput;
 import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.util.IntsRef;
 import org.apache.lucene.util.IntsRefBuilder;
 import org.apache.lucene.util.UnicodeUtil;
 import org.apache.lucene.util.fst.FST;
 import org.apache.lucene.util.fst.IntsRefFSTEnum;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
 
 /** Tests of TokenInfoDictionary build tools; run using ant test-tools */
 public class TestTokenInfoDictionary extends LuceneTestCase {
@@ -185,4 +189,25 @@ public class TestTokenInfoDictionary extends LuceneTestCase {
       System.out.println("checked " + numTerms + " terms, " + numWords + " words.");
     }
   }
+
+  // #12911: make sure our shipped binary FST for TokenInfoDictionary is the latest & greatest
+  // format
+  public void testBinaryFSTIsLatestFormat() throws Exception {
+    try (InputStream is =
+        new BufferedInputStream(
+            TokenInfoDictionary.getClassResource(TokenInfoDictionary.FST_FILENAME_SUFFIX))) {
+      // we only need to load the FSTMetadata to check version:
+      int actualVersion =
+          FST.readMetadata(new InputStreamDataInput(is), PositiveIntOutputs.getSingleton())
+              .getVersion();
+      assertEquals(
+          "TokenInfoDictionary's FST is not the latest version: expected "
+              + FST.VERSION_CURRENT
+              + " but got: "
+              + actualVersion
+              + "; run \"./gradlew :lucene:analysis:nori:regenerate\" to regenerate this FST",
+          FST.VERSION_CURRENT,
+          actualVersion);
+    }
+  }
 }
diff --git a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
index cc161ba3f8d..260c159f207 100644
--- a/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
+++ b/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
@@ -1208,7 +1208,7 @@ public final class FST<T> implements Accountable {
   }
 
   /**
-   * Represent the FST metadata
+   * Represents the FST metadata.
    *
    * @param <T> the FST output type
    */
@@ -1236,5 +1236,14 @@ public final class FST<T> implements Accountable {
       this.version = version;
       this.numBytes = numBytes;
     }
+
+    /**
+     * Returns the version constant of the binary format this FST was written in. See the {@code
+     * static final int VERSION} constants in FST's javadoc, e.g. {@link
+     * FST#VERSION_CONTINUOUS_ARCS}.
+     */
+    public int getVersion() {
+      return version;
+    }
   }
 }