You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2025/09/27 23:23:45 UTC

(lucene) 01/02: Fix SmartChinese to only deserialize data from classpath with a native array filter applied and never cache dictionaries from custom locations (#15237)

This is an automated email from the ASF dual-hosted git repository.

uschindler pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git

commit 6c9c2df54708dcae7e38534f6079deb9223ef59d
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Sun Sep 28 00:27:57 2025 +0200

    Fix SmartChinese to only deserialize data from classpath with a native array filter applied and never cache dictionaries from custom locations (#15237)
    
    * Fix SmartChinese to only serialize data from classpath with a native array filter applied and never cache dictionaries from custom locations
    
    * fix exception handling
    
    * add CHANGES.txt
    
    * fix exception handling
    
    * fix typo in CHANGES.txt
    
    * Restore the code to regenerate the serialized file
    
    * Disallow any serialization in test-framework as early as possible (we can't do that via sysprops due to Gradle)
    
    * Install a serialization filter that only allows Gradle's test runner config deserialization
    
    * fix errorprone
    
    * use same logic like for security manager to install the filter
    
    * add new filter to CHANGES.txt
    
    * Improve performance by only installing a filter if we're not called from Gradle
    
    * simplify
    
    * Add a test for the deserialization filter
    
    * fix typo
    
    * improve test
    # Conflicts:
    #       build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java
    #       lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
---
 gradle/testing/randomization.gradle                |  8 ++
 gradle/testing/randomization/policies/tests.policy |  3 +
 lucene/CHANGES.txt                                 |  5 +-
 .../analysis/cn/smart/hhmm/AbstractDictionary.java | 15 ++++
 .../analysis/cn/smart/hhmm/BigramDictionary.java   | 47 +++++-------
 .../analysis/cn/smart/hhmm/WordDictionary.java     | 88 +++++++++++-----------
 .../tests/util/TestObjectInputFilterFactory.java   | 55 ++++++++++++++
 .../util/TestTestObjectInputFilterFactory.java     | 79 +++++++++++++++++++
 8 files changed, 226 insertions(+), 74 deletions(-)

diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle
index 185cd0872a9..91280058be5 100644
--- a/gradle/testing/randomization.gradle
+++ b/gradle/testing/randomization.gradle
@@ -218,6 +218,14 @@ allprojects {
           securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath)
         }
 
+        // disallow any Java serialization without a filter
+        if (project.path.endsWith(".tests")) {
+          // LUCENE-10301: for now, do not use the serialization filter for modular tests
+          // (test framework is not available).
+        } else if (project.path.startsWith(":lucene")) {
+          systemProperty 'jdk.serialFilterFactory', 'org.apache.lucene.tests.util.TestObjectInputFilterFactory'
+        }
+
         doFirst {
           logger.debug("Will use test opts:\n" + testOptionsResolved.collect {k,v -> "${k}: ${v}"}.sort().join("\n"))
         }
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy
index f8e09ba0366..822c9988cee 100644
--- a/gradle/testing/randomization/policies/tests.policy
+++ b/gradle/testing/randomization/policies/tests.policy
@@ -80,6 +80,9 @@ grant {
   permission java.io.FilePermission "${hunspell.corpora}${/}-", "read";
   permission java.io.FilePermission "${hunspell.dictionaries}", "read";
   permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";
+  
+  // deserialization in SmartChinese needs to be safe.
+  permission java.io.SerializablePermission "serialFilter";
 };
 
 // Permissions for jacoco code coverage
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6927189f049..f0551f321c3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -70,7 +70,10 @@ Bug Fixes
 
 Other
 ---------------------
-(No changes)
+* GITHUB#15237: Fix SmartChinese to only deserialize dictionary data from classpath with a native array
+  filter applied and never cache dictionaries from custom locations. Also install a global deserialization
+  filter factory that disallows any unprotected deserialization while running tests.
+  (Uwe Schindler, Isaac David)
 
 ======================= Lucene 10.3.0 =======================
 
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java
index 28a98502d25..9aade3d1a60 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java
@@ -16,6 +16,9 @@
  */
 package org.apache.lucene.analysis.cn.smart.hhmm;
 
+import java.io.ObjectInputFilter.FilterInfo;
+import java.io.ObjectInputFilter.Status;
+import java.io.ObjectInputStream;
 import java.io.UnsupportedEncodingException;
 
 /**
@@ -197,4 +200,16 @@ abstract class AbstractDictionary {
 
     return hash;
   }
+
+  /** Safety filter for {@link ObjectInputStream} to only allow arrays of primitive types. */
+  final Status filterObjectInputStream(FilterInfo info) {
+    var cl = info.serialClass();
+    if (cl == null) {
+      return Status.UNDECIDED;
+    }
+    while (cl.isArray()) {
+      cl = cl.getComponentType();
+    }
+    return cl.isPrimitive() ? Status.ALLOWED : Status.REJECTED;
+  }
 }
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
index 885cc144037..c4cce0db428 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
@@ -26,6 +26,8 @@ import java.nio.ByteOrder;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
 import org.apache.lucene.util.SuppressForbidden;
 
@@ -77,20 +79,18 @@ class BigramDictionary extends AbstractDictionary {
     return singleInstance;
   }
 
-  private boolean loadFromObj(Path serialObj) {
-    try {
-      loadFromInputStream(Files.newInputStream(serialObj));
-      return true;
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   @SuppressForbidden(
       reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")
+  @SuppressWarnings("removal")
   private void loadFromInputStream(InputStream serialObjectInputStream)
       throws IOException, ClassNotFoundException {
     try (ObjectInputStream input = new ObjectInputStream(serialObjectInputStream)) {
+      AccessController.doPrivileged(
+          (PrivilegedAction<Void>)
+              () -> {
+                input.setObjectInputFilter(this::filterObjectInputStream);
+                return null;
+              });
       bigramHashTable = (long[]) input.readObject();
       frequencyTable = (int[]) input.readObject();
       // log.info("load bigram dict from serialization.");
@@ -114,26 +114,17 @@ class BigramDictionary extends AbstractDictionary {
 
   private void load(String dictRoot) throws IOException {
     String bigramDictPath = dictRoot + "/bigramdict.dct";
-
-    Path serialObj = Paths.get(dictRoot + "/bigramdict.mem");
-
-    if (Files.exists(serialObj) && loadFromObj(serialObj)) {
-
-    } else {
-      try {
-        bigramHashTable = new long[PRIME_BIGRAM_LENGTH];
-        frequencyTable = new int[PRIME_BIGRAM_LENGTH];
-        for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {
-          // it is possible for a value to hash to 0, but the probability is extremely low
-          bigramHashTable[i] = 0;
-          frequencyTable[i] = 0;
-        }
-        loadFromFile(bigramDictPath);
-      } catch (IOException e) {
-        throw new RuntimeException(e.getMessage());
-      }
-      saveToObj(serialObj);
+    bigramHashTable = new long[PRIME_BIGRAM_LENGTH];
+    frequencyTable = new int[PRIME_BIGRAM_LENGTH];
+    for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {
+      // it is possible for a value to hash to 0, but the probability is extremely low
+      bigramHashTable[i] = 0;
+      frequencyTable[i] = 0;
     }
+    loadFromFile(bigramDictPath);
+
+    /* Enable the following line to regenerate the serialized file: */
+    // saveToObj(Paths.get(dictRoot + "/bigramdict.mem"));
   }
 
   /**
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
index cba7e3d887e..8c08f6b38b8 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
@@ -26,6 +26,8 @@ import java.nio.ByteOrder;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
 import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
 import org.apache.lucene.analysis.cn.smart.Utility;
 import org.apache.lucene.util.SuppressForbidden;
@@ -81,9 +83,21 @@ class WordDictionary extends AbstractDictionary {
         singleInstance.load();
       } catch (
           @SuppressWarnings("unused")
-          IOException e) {
-        String wordDictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
-        singleInstance.load(wordDictRoot);
+          IOException e0) {
+        try {
+          singleInstance.load();
+        } catch (IOException e) {
+          String dictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
+          try {
+            singleInstance.load(dictRoot);
+          } catch (IOException ioe) {
+            RuntimeException ex = new RuntimeException(ioe);
+            ex.addSuppressed(e);
+            throw ex;
+          }
+        } catch (ClassNotFoundException e) {
+          throw new RuntimeException(e);
+        }
       } catch (ClassNotFoundException e) {
         throw new RuntimeException(e);
       }
@@ -92,39 +106,29 @@ class WordDictionary extends AbstractDictionary {
   }
 
   /**
-   * Attempt to load dictionary from provided directory, first trying coredict.mem, failing back on
-   * coredict.dct
+   * Attempt to load dictionary from provided path to coredict.dct
    *
    * @param dctFileRoot path to dictionary directory
    */
-  public void load(String dctFileRoot) {
+  public void load(String dctFileRoot) throws IOException {
     String dctFilePath = dctFileRoot + "/coredict.dct";
-    Path serialObj = Paths.get(dctFileRoot + "/coredict.mem");
-
-    if (Files.exists(serialObj) && loadFromObj(serialObj)) {
-
-    } else {
-      try {
-        wordIndexTable = new short[PRIME_INDEX_LENGTH];
-        charIndexTable = new char[PRIME_INDEX_LENGTH];
-        for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {
-          charIndexTable[i] = 0;
-          wordIndexTable[i] = -1;
-        }
-        wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];
-        wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];
-        // int total =
-        loadMainDataFromFile(dctFilePath);
-        expandDelimiterData();
-        mergeSameWords();
-        sortEachItems();
-        // log.info("load dictionary: " + dctFilePath + " total:" + total);
-      } catch (IOException e) {
-        throw new RuntimeException(e.getMessage());
-      }
-
-      saveToObj(serialObj);
+    wordIndexTable = new short[PRIME_INDEX_LENGTH];
+    charIndexTable = new char[PRIME_INDEX_LENGTH];
+    for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {
+      charIndexTable[i] = 0;
+      wordIndexTable[i] = -1;
     }
+    wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];
+    wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];
+    // int total =
+    loadMainDataFromFile(dctFilePath);
+    expandDelimiterData();
+    mergeSameWords();
+    sortEachItems();
+    // log.info("load dictionary: " + dctFilePath + " total:" + total);
+
+    /* Enable the following line to regenerate the serialized file: */
+    // saveToObj(Paths.get(dctFileRoot + "/coredict.mem"));
   }
 
   /**
@@ -137,20 +141,18 @@ class WordDictionary extends AbstractDictionary {
     loadFromObjectInputStream(input);
   }
 
-  private boolean loadFromObj(Path serialObj) {
-    try {
-      loadFromObjectInputStream(Files.newInputStream(serialObj));
-      return true;
-    } catch (Exception e) {
-      throw new RuntimeException(e);
-    }
-  }
-
   @SuppressForbidden(
       reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")
+  @SuppressWarnings("removal")
   private void loadFromObjectInputStream(InputStream serialObjectInputStream)
       throws IOException, ClassNotFoundException {
     try (ObjectInputStream input = new ObjectInputStream(serialObjectInputStream)) {
+      AccessController.doPrivileged(
+          (PrivilegedAction<Void>)
+              () -> {
+                input.setObjectInputFilter(this::filterObjectInputStream);
+                return null;
+              });
       wordIndexTable = (short[]) input.readObject();
       charIndexTable = (char[]) input.readObject();
       wordItem_charArrayTable = (char[][][]) input.readObject();
@@ -161,17 +163,13 @@ class WordDictionary extends AbstractDictionary {
 
   @SuppressForbidden(
       reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")
-  private void saveToObj(Path serialObj) {
+  private void saveToObj(Path serialObj) throws IOException {
     try (ObjectOutputStream output = new ObjectOutputStream(Files.newOutputStream(serialObj))) {
       output.writeObject(wordIndexTable);
       output.writeObject(charIndexTable);
       output.writeObject(wordItem_charArrayTable);
       output.writeObject(wordItem_frequencyTable);
       // log.info("serialize core dict.");
-    } catch (
-        @SuppressWarnings("unused")
-        Exception e) {
-      // log.warn(e.getMessage());
     }
   }
 
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java
new file mode 100644
index 00000000000..53ef2a2cc00
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.tests.util;
+
+import java.io.ObjectInputFilter;
+import java.io.ObjectInputFilter.Status;
+import java.lang.StackWalker.StackFrame;
+import java.util.function.BinaryOperator;
+
+/**
+ * Make sure we do not allow Java serialization without custom filters anywhere while running tests.
+ * This factory returns a "deny-all" deserialization filter if none was installed by the
+ * deserializer. To allow Gradle's test runner to serialize its config, all calls from Gradle are
+ * allowed.
+ */
+public final class TestObjectInputFilterFactory implements BinaryOperator<ObjectInputFilter> {
+
+  /** An {@link ObjectInputFilter} that rejects any deserialization. */
+  public static final ObjectInputFilter DENY_ALL_FILTER = info -> Status.REJECTED;
+
+  @Override
+  public ObjectInputFilter apply(ObjectInputFilter curr, ObjectInputFilter next) {
+    // if Gradle's deserializer is on the stack, return next filter (which is obviously null):
+    if (StackWalker.getInstance()
+        .walk(s -> s.anyMatch(TestObjectInputFilterFactory::isGradleSerializerStackFrame))) {
+      return next;
+    }
+    // if no filter was configured, return our filter that rejects all deserialization:
+    if (curr == null && next == null) {
+      return DENY_ALL_FILTER;
+    }
+    // if code installs its own filter return it:
+    return next;
+  }
+
+  private static boolean isGradleSerializerStackFrame(StackFrame f) {
+    final String methodName = f.getMethodName(), className = f.getClassName();
+    return ("deserializeWorker".equals(methodName)
+        && "org.gradle.process.internal.worker.messaging.WorkerConfigSerializer".equals(className));
+  }
+}
diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java
new file mode 100644
index 00000000000..2d9da257bdb
--- /dev/null
+++ b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.tests.util;
+
+import static java.io.ObjectInputFilter.Config.createFilter;
+import static java.io.ObjectInputFilter.Config.getSerialFilterFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InvalidClassException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import org.apache.lucene.util.SuppressForbidden;
+import org.junit.BeforeClass;
+
+@SuppressForbidden(reason = "tests the forbidden code")
+public final class TestTestObjectInputFilterFactory extends LuceneTestCase {
+
+  @BeforeClass
+  public static void beforeClass() {
+    assumeTrue(
+        "Only works when TestObjectInputFilterFactory is installed in JVM",
+        getSerialFilterFactory() instanceof TestObjectInputFilterFactory);
+  }
+
+  private InputStream openStream() throws IOException {
+    var out = new ByteArrayOutputStream();
+    try (var oout = new ObjectOutputStream(out)) {
+      oout.writeObject(new HashMap<>());
+    }
+    return new ByteArrayInputStream(out.toByteArray());
+  }
+
+  public void testDeserializationWithoutFilter() throws Exception {
+    try (var oin = new ObjectInputStream(openStream())) {
+      // no filter set!
+      var ex = assertThrows(InvalidClassException.class, () -> oin.readObject());
+      assertTrue(ex.getMessage().contains("REJECTED"));
+    }
+  }
+
+  public void testDeserializationWithAllowFilter() throws Exception {
+    try (var oin = new ObjectInputStream(openStream())) {
+      oin.setObjectInputFilter(createFilter("java.util.HashMap;!*"));
+      oin.readObject();
+    }
+  }
+
+  public void testDeserializationWithDenyFilter() throws Exception {
+    try (var oin = new ObjectInputStream(openStream())) {
+      oin.setObjectInputFilter(createFilter("!*"));
+      var ex = assertThrows(InvalidClassException.class, () -> oin.readObject());
+      assertTrue(ex.getMessage().contains("REJECTED"));
+    }
+  }
+
+  public void testBadCodeJustSettingNullFilter() throws Exception {
+    try (var oin = new ObjectInputStream(openStream())) {
+      assertThrows(IllegalStateException.class, () -> oin.setObjectInputFilter(null));
+    }
+  }
+}