You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2025/09/27 23:23:45 UTC
(lucene) 01/02: Fix SmartChinese to only deserialize data from classpath with a native array filter applied and never cache dictionaries from custom locations (#15237)
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 6c9c2df54708dcae7e38534f6079deb9223ef59d
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Sun Sep 28 00:27:57 2025 +0200
Fix SmartChinese to only deserialize data from classpath with a native array filter applied and never cache dictionaries from custom locations (#15237)
* Fix SmartChinese to only serialize data from classpath with a native array filter applied and never cache dictionaries from custom locations
* fix exception handling
* add CHANGES.txt
* fix exception handling
* fix typo in CHANGES.txt
* Restore the code to regenerate the serialized file
* Disallow any serialization in test-framework as early as possible (we can't do that via sysprops due to Gradle)
* Install a serialization filter that only allows Gradle's test runner config deserialization
* fix errorprone
* use same logic like for security manager to install the filter
* add new filter to CHANGES.txt
* Improve performance by only installing a filter if we're not called from Gradle
* simplify
* Add a test for the deserialization filter
* fix typo
* improve test
# Conflicts:
# build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java
# lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
---
gradle/testing/randomization.gradle | 8 ++
gradle/testing/randomization/policies/tests.policy | 3 +
lucene/CHANGES.txt | 5 +-
.../analysis/cn/smart/hhmm/AbstractDictionary.java | 15 ++++
.../analysis/cn/smart/hhmm/BigramDictionary.java | 47 +++++-------
.../analysis/cn/smart/hhmm/WordDictionary.java | 88 +++++++++++-----------
.../tests/util/TestObjectInputFilterFactory.java | 55 ++++++++++++++
.../util/TestTestObjectInputFilterFactory.java | 79 +++++++++++++++++++
8 files changed, 226 insertions(+), 74 deletions(-)
diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle
index 185cd0872a9..91280058be5 100644
--- a/gradle/testing/randomization.gradle
+++ b/gradle/testing/randomization.gradle
@@ -218,6 +218,14 @@ allprojects {
securityArgumentProvider.otherProperties.put("lucene.test.framework", project(":lucene:test-framework").layout.buildDirectory.get().asFile.absolutePath)
}
+ // disallow any Java serialization without a filter
+ if (project.path.endsWith(".tests")) {
+ // LUCENE-10301: for now, do not use the serialization filter for modular tests
+ // (test framework is not available).
+ } else if (project.path.startsWith(":lucene")) {
+ systemProperty 'jdk.serialFilterFactory', 'org.apache.lucene.tests.util.TestObjectInputFilterFactory'
+ }
+
doFirst {
logger.debug("Will use test opts:\n" + testOptionsResolved.collect {k,v -> "${k}: ${v}"}.sort().join("\n"))
}
diff --git a/gradle/testing/randomization/policies/tests.policy b/gradle/testing/randomization/policies/tests.policy
index f8e09ba0366..822c9988cee 100644
--- a/gradle/testing/randomization/policies/tests.policy
+++ b/gradle/testing/randomization/policies/tests.policy
@@ -80,6 +80,9 @@ grant {
permission java.io.FilePermission "${hunspell.corpora}${/}-", "read";
permission java.io.FilePermission "${hunspell.dictionaries}", "read";
permission java.io.FilePermission "${hunspell.dictionaries}${/}-", "read";
+
+ // deserialization in SmartChinese needs to be safe.
+ permission java.io.SerializablePermission "serialFilter";
};
// Permissions for jacoco code coverage
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 6927189f049..f0551f321c3 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -70,7 +70,10 @@ Bug Fixes
Other
---------------------
-(No changes)
+* GITHUB#15237: Fix SmartChinese to only deserialize dictionary data from classpath with a native array
+ filter applied and never cache dictionaries from custom locations. Also install a global deserialization
+ filter factory that disallows any unprotected deserialization while running tests.
+ (Uwe Schindler, Isaac David)
======================= Lucene 10.3.0 =======================
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java
index 28a98502d25..9aade3d1a60 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java
@@ -16,6 +16,9 @@
*/
package org.apache.lucene.analysis.cn.smart.hhmm;
+import java.io.ObjectInputFilter.FilterInfo;
+import java.io.ObjectInputFilter.Status;
+import java.io.ObjectInputStream;
import java.io.UnsupportedEncodingException;
/**
@@ -197,4 +200,16 @@ abstract class AbstractDictionary {
return hash;
}
+
+ /** Safety filter for {@link ObjectInputStream} to only allow arrays of primitive types. */
+ final Status filterObjectInputStream(FilterInfo info) {
+ var cl = info.serialClass();
+ if (cl == null) {
+ return Status.UNDECIDED;
+ }
+ while (cl.isArray()) {
+ cl = cl.getComponentType();
+ }
+ return cl.isPrimitive() ? Status.ALLOWED : Status.REJECTED;
+ }
}
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
index 885cc144037..c4cce0db428 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
@@ -26,6 +26,8 @@ import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
import org.apache.lucene.util.SuppressForbidden;
@@ -77,20 +79,18 @@ class BigramDictionary extends AbstractDictionary {
return singleInstance;
}
- private boolean loadFromObj(Path serialObj) {
- try {
- loadFromInputStream(Files.newInputStream(serialObj));
- return true;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
@SuppressForbidden(
reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")
+ @SuppressWarnings("removal")
private void loadFromInputStream(InputStream serialObjectInputStream)
throws IOException, ClassNotFoundException {
try (ObjectInputStream input = new ObjectInputStream(serialObjectInputStream)) {
+ AccessController.doPrivileged(
+ (PrivilegedAction<Void>)
+ () -> {
+ input.setObjectInputFilter(this::filterObjectInputStream);
+ return null;
+ });
bigramHashTable = (long[]) input.readObject();
frequencyTable = (int[]) input.readObject();
// log.info("load bigram dict from serialization.");
@@ -114,26 +114,17 @@ class BigramDictionary extends AbstractDictionary {
private void load(String dictRoot) throws IOException {
String bigramDictPath = dictRoot + "/bigramdict.dct";
-
- Path serialObj = Paths.get(dictRoot + "/bigramdict.mem");
-
- if (Files.exists(serialObj) && loadFromObj(serialObj)) {
-
- } else {
- try {
- bigramHashTable = new long[PRIME_BIGRAM_LENGTH];
- frequencyTable = new int[PRIME_BIGRAM_LENGTH];
- for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {
- // it is possible for a value to hash to 0, but the probability is extremely low
- bigramHashTable[i] = 0;
- frequencyTable[i] = 0;
- }
- loadFromFile(bigramDictPath);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage());
- }
- saveToObj(serialObj);
+ bigramHashTable = new long[PRIME_BIGRAM_LENGTH];
+ frequencyTable = new int[PRIME_BIGRAM_LENGTH];
+ for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {
+ // it is possible for a value to hash to 0, but the probability is extremely low
+ bigramHashTable[i] = 0;
+ frequencyTable[i] = 0;
}
+ loadFromFile(bigramDictPath);
+
+ /* Enable the following line to regenerate the serialized file: */
+ // saveToObj(Paths.get(dictRoot + "/bigramdict.mem"));
}
/**
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
index cba7e3d887e..8c08f6b38b8 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
@@ -26,6 +26,8 @@ import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
import org.apache.lucene.analysis.cn.smart.Utility;
import org.apache.lucene.util.SuppressForbidden;
@@ -81,9 +83,21 @@ class WordDictionary extends AbstractDictionary {
singleInstance.load();
} catch (
@SuppressWarnings("unused")
- IOException e) {
- String wordDictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
- singleInstance.load(wordDictRoot);
+ IOException e0) {
+ try {
+ singleInstance.load();
+ } catch (IOException e) {
+ String dictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;
+ try {
+ singleInstance.load(dictRoot);
+ } catch (IOException ioe) {
+ RuntimeException ex = new RuntimeException(ioe);
+ ex.addSuppressed(e);
+ throw ex;
+ }
+ } catch (ClassNotFoundException e) {
+ throw new RuntimeException(e);
+ }
} catch (ClassNotFoundException e) {
throw new RuntimeException(e);
}
@@ -92,39 +106,29 @@ class WordDictionary extends AbstractDictionary {
}
/**
- * Attempt to load dictionary from provided directory, first trying coredict.mem, failing back on
- * coredict.dct
+ * Attempt to load dictionary from provided path to coredict.dct
*
* @param dctFileRoot path to dictionary directory
*/
- public void load(String dctFileRoot) {
+ public void load(String dctFileRoot) throws IOException {
String dctFilePath = dctFileRoot + "/coredict.dct";
- Path serialObj = Paths.get(dctFileRoot + "/coredict.mem");
-
- if (Files.exists(serialObj) && loadFromObj(serialObj)) {
-
- } else {
- try {
- wordIndexTable = new short[PRIME_INDEX_LENGTH];
- charIndexTable = new char[PRIME_INDEX_LENGTH];
- for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {
- charIndexTable[i] = 0;
- wordIndexTable[i] = -1;
- }
- wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];
- wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];
- // int total =
- loadMainDataFromFile(dctFilePath);
- expandDelimiterData();
- mergeSameWords();
- sortEachItems();
- // log.info("load dictionary: " + dctFilePath + " total:" + total);
- } catch (IOException e) {
- throw new RuntimeException(e.getMessage());
- }
-
- saveToObj(serialObj);
+ wordIndexTable = new short[PRIME_INDEX_LENGTH];
+ charIndexTable = new char[PRIME_INDEX_LENGTH];
+ for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {
+ charIndexTable[i] = 0;
+ wordIndexTable[i] = -1;
}
+ wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];
+ wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];
+ // int total =
+ loadMainDataFromFile(dctFilePath);
+ expandDelimiterData();
+ mergeSameWords();
+ sortEachItems();
+ // log.info("load dictionary: " + dctFilePath + " total:" + total);
+
+ /* Enable the following line to regenerate the serialized file: */
+ // saveToObj(Paths.get(dctFileRoot + "/coredict.mem"));
}
/**
@@ -137,20 +141,18 @@ class WordDictionary extends AbstractDictionary {
loadFromObjectInputStream(input);
}
- private boolean loadFromObj(Path serialObj) {
- try {
- loadFromObjectInputStream(Files.newInputStream(serialObj));
- return true;
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
- }
-
@SuppressForbidden(
reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")
+ @SuppressWarnings("removal")
private void loadFromObjectInputStream(InputStream serialObjectInputStream)
throws IOException, ClassNotFoundException {
try (ObjectInputStream input = new ObjectInputStream(serialObjectInputStream)) {
+ AccessController.doPrivileged(
+ (PrivilegedAction<Void>)
+ () -> {
+ input.setObjectInputFilter(this::filterObjectInputStream);
+ return null;
+ });
wordIndexTable = (short[]) input.readObject();
charIndexTable = (char[]) input.readObject();
wordItem_charArrayTable = (char[][][]) input.readObject();
@@ -161,17 +163,13 @@ class WordDictionary extends AbstractDictionary {
@SuppressForbidden(
reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")
- private void saveToObj(Path serialObj) {
+ private void saveToObj(Path serialObj) throws IOException {
try (ObjectOutputStream output = new ObjectOutputStream(Files.newOutputStream(serialObj))) {
output.writeObject(wordIndexTable);
output.writeObject(charIndexTable);
output.writeObject(wordItem_charArrayTable);
output.writeObject(wordItem_frequencyTable);
// log.info("serialize core dict.");
- } catch (
- @SuppressWarnings("unused")
- Exception e) {
- // log.warn(e.getMessage());
}
}
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java
new file mode 100644
index 00000000000..53ef2a2cc00
--- /dev/null
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.tests.util;
+
+import java.io.ObjectInputFilter;
+import java.io.ObjectInputFilter.Status;
+import java.lang.StackWalker.StackFrame;
+import java.util.function.BinaryOperator;
+
+/**
+ * Make sure we do not allow Java serialization without custom filters anywhere while running tests.
+ * This factory returns a "deny-all" deserialization filter if none was installed by the
+ * deserializer. To allow Gradle's test runner to serialize its config, all calls from Gradle are
+ * allowed.
+ */
+public final class TestObjectInputFilterFactory implements BinaryOperator<ObjectInputFilter> {
+
+ /** An {@link ObjectInputFilter} that rejects any deserialization. */
+ public static final ObjectInputFilter DENY_ALL_FILTER = info -> Status.REJECTED;
+
+ @Override
+ public ObjectInputFilter apply(ObjectInputFilter curr, ObjectInputFilter next) {
+ // if Gradle's deserializer is on the stack, return next filter (which is obviously null):
+ if (StackWalker.getInstance()
+ .walk(s -> s.anyMatch(TestObjectInputFilterFactory::isGradleSerializerStackFrame))) {
+ return next;
+ }
+ // if no filter was configured, return our filter that rejects all deserialization:
+ if (curr == null && next == null) {
+ return DENY_ALL_FILTER;
+ }
+ // if code installs its own filter return it:
+ return next;
+ }
+
+ private static boolean isGradleSerializerStackFrame(StackFrame f) {
+ final String methodName = f.getMethodName(), className = f.getClassName();
+ return ("deserializeWorker".equals(methodName)
+ && "org.gradle.process.internal.worker.messaging.WorkerConfigSerializer".equals(className));
+ }
+}
diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java
new file mode 100644
index 00000000000..2d9da257bdb
--- /dev/null
+++ b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.tests.util;
+
+import static java.io.ObjectInputFilter.Config.createFilter;
+import static java.io.ObjectInputFilter.Config.getSerialFilterFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InvalidClassException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import org.apache.lucene.util.SuppressForbidden;
+import org.junit.BeforeClass;
+
+@SuppressForbidden(reason = "tests the forbidden code")
+public final class TestTestObjectInputFilterFactory extends LuceneTestCase {
+
+ @BeforeClass
+ public static void beforeClass() {
+ assumeTrue(
+ "Only works when TestObjectInputFilterFactory is installed in JVM",
+ getSerialFilterFactory() instanceof TestObjectInputFilterFactory);
+ }
+
+ private InputStream openStream() throws IOException {
+ var out = new ByteArrayOutputStream();
+ try (var oout = new ObjectOutputStream(out)) {
+ oout.writeObject(new HashMap<>());
+ }
+ return new ByteArrayInputStream(out.toByteArray());
+ }
+
+ public void testDeserializationWithoutFilter() throws Exception {
+ try (var oin = new ObjectInputStream(openStream())) {
+ // no filter set!
+ var ex = assertThrows(InvalidClassException.class, () -> oin.readObject());
+ assertTrue(ex.getMessage().contains("REJECTED"));
+ }
+ }
+
+ public void testDeserializationWithAllowFilter() throws Exception {
+ try (var oin = new ObjectInputStream(openStream())) {
+ oin.setObjectInputFilter(createFilter("java.util.HashMap;!*"));
+ oin.readObject();
+ }
+ }
+
+ public void testDeserializationWithDenyFilter() throws Exception {
+ try (var oin = new ObjectInputStream(openStream())) {
+ oin.setObjectInputFilter(createFilter("!*"));
+ var ex = assertThrows(InvalidClassException.class, () -> oin.readObject());
+ assertTrue(ex.getMessage().contains("REJECTED"));
+ }
+ }
+
+ public void testBadCodeJustSettingNullFilter() throws Exception {
+ try (var oin = new ObjectInputStream(openStream())) {
+ assertThrows(IllegalStateException.class, () -> oin.setObjectInputFilter(null));
+ }
+ }
+}