You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ho...@apache.org on 2026/09/26 02:58:47 UTC
(spark) branch branch-4.3 updated: [SPARK-59333][CORE] Apply a serialization filter when recovering Master state from ZooKeeper
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new 44f3073ca725 [SPARK-59333][CORE] Apply a serialization filter when recovering Master state from ZooKeeper
44f3073ca725 is described below
commit 44f3073ca7257c9df6603c71b6cff9d0d24c22bd
Author: Holden Karau <ho...@pigscanfly.ca>
AuthorDate: Fri Sep 25 19:57:53 2026 -0700
[SPARK-59333][CORE] Apply a serialization filter when recovering Master state from ZooKeeper
### What changes were proposed in this pull request?
Harden deserialization when the standalone master recovers state from ZooKeeper.
This is done by a new config `spark.deploy.recoverySerializationFilter` (default
`java.**;scala.**;org.apache.spark.**;!*`, which covers `ApplicationInfo`, `DriverInfo`,
`WorkerInfo` and their fields). `*` disables filtering; values that yield no filter (empty,
`;`) are rejected when the config is read. Znodes rejected by the filter are skipped, not
deleted.
This does not replace the need for ZK ACLs, which remain the access control for the recovery
state.
### Why are the changes needed?
If we're recovering from failure, the recovery state may contain unexpected classes;
restricting what gets instantiated limits what a corrupted or unexpected znode can do.
### Does this PR introduce _any_ user-facing change?
Configurable filtering on classes during ZK recovery using `spark.deploy.recoverySerializationFilter`.
### How was this patch tested?
New unit test in `PersistenceEngineSuite`
### Was this patch authored or co-authored using generative AI tooling?
Yes
Generated-by: Claude (mixture of version, mostly Opus 5) and Cursor (Kimi K3)
Closes #58620 from holdenk/zk-recovery-data-validation.
Lead-authored-by: Holden Karau <ho...@pigscanfly.ca>
Co-authored-by: Holden Karau <ho...@snowflake.com>
Signed-off-by: Holden Karau <ho...@snowflake.com>
(cherry picked from commit 414f7384487bf02c0a14c991ac6746ff8ecc916f)
Signed-off-by: Holden Karau <ho...@snowflake.com>
---
.../master/FileSystemPersistenceEngine.scala | 3 +
.../deploy/master/RocksDBPersistenceEngine.scala | 3 +
.../deploy/master/ZooKeeperPersistenceEngine.scala | 49 +++++++++++++-
.../org/apache/spark/internal/config/Deploy.scala | 21 ++++++
.../apache/spark/serializer/JavaSerializer.scala | 20 +++++-
.../deploy/master/PersistenceEngineSuite.scala | 77 +++++++++++++++++++++-
docs/spark-standalone.md | 18 +++++
7 files changed, 185 insertions(+), 6 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
index 0c9baf9e6f70..6df3c188def7 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/FileSystemPersistenceEngine.scala
@@ -84,6 +84,9 @@ private[master] class FileSystemPersistenceEngine(
}
}
+ // Unlike ZooKeeperPersistenceEngine, no recovery serialization filter is applied here:
+ // the store is a lower-exposure surface than a shared ZooKeeper ensemble, though it can
+ // still be shared across hosts (e.g. an NFS-mounted recovery directory).
private def deserializeFromFile[T](file: File)(implicit m: ClassTag[T]): T = {
var fileIn: InputStream = new FileInputStream(file)
codec.foreach { c => fileIn = c.compressedInputStream(new FileInputStream(file)) }
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
index 8364dbd693b1..006cd92c2c4c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/RocksDBPersistenceEngine.scala
@@ -91,6 +91,9 @@ private[master] class RocksDBPersistenceEngine(
db.delete(name.getBytes(UTF_8))
}
+ // Unlike ZooKeeperPersistenceEngine, no recovery serialization filter is applied here:
+ // the store is local to the master host; if it is corrupted, the master cannot trust
+ // itself anyway.
override def read[T: ClassTag](name: String): Seq[T] = {
val result = new ArrayBuffer[T]
val iter = db.newIterator()
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
index d02f09e18359..ed5121585eea 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/ZooKeeperPersistenceEngine.scala
@@ -17,6 +17,7 @@
package org.apache.spark.deploy.master
+import java.io.ObjectInputFilter
import java.nio.ByteBuffer
import scala.jdk.CollectionConverters._
@@ -27,9 +28,10 @@ import org.apache.zookeeper.CreateMode
import org.apache.spark.SparkConf
import org.apache.spark.deploy.SparkCuratorUtil
-import org.apache.spark.internal.Logging
+import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.Deploy._
-import org.apache.spark.serializer.Serializer
+import org.apache.spark.serializer.{JavaSerializerInstance, Serializer}
+import org.apache.spark.util.ByteBufferInputStream
private[master] class ZooKeeperPersistenceEngine(conf: SparkConf, val serializer: Serializer)
@@ -39,6 +41,11 @@ private[master] class ZooKeeperPersistenceEngine(conf: SparkConf, val serializer
private val workingDir = conf.get(ZOOKEEPER_DIRECTORY).getOrElse("/spark") + "/master_status"
private val zk: CuratorFramework = SparkCuratorUtil.newClient(conf)
+ // Only instantiate well-known classes while reading persisted state back, so corrupted
+ // or unexpected znode contents are never instantiated in the newly elected master.
+ private val serializationFilter: ObjectInputFilter =
+ ObjectInputFilter.Config.createFilter(conf.get(RECOVERY_SERIALIZATION_FILTER))
+
SparkCuratorUtil.mkdir(zk, workingDir)
@@ -68,13 +75,49 @@ private[master] class ZooKeeperPersistenceEngine(conf: SparkConf, val serializer
private def deserializeFromFile[T](filename: String)(implicit m: ClassTag[T]): Option[T] = {
val fileData = zk.getData().forPath(workingDir + "/" + filename)
+ val recordingFilter = new RecordingFilter(serializationFilter)
try {
- Some(serializer.newInstance().deserialize[T](ByteBuffer.wrap(fileData)))
+ serializer.newInstance() match {
+ case javaInstance: JavaSerializerInstance =>
+ val in = javaInstance.deserializeStream(
+ new ByteBufferInputStream(ByteBuffer.wrap(fileData)), recordingFilter)
+ try {
+ Some(in.readObject[T]())
+ } finally {
+ in.close()
+ }
+ case instance =>
+ Some(instance.deserialize[T](ByteBuffer.wrap(fileData)))
+ }
} catch {
+ case e: Exception if recordingFilter.rejected =>
+ // Rejected by the serialization filter, not found corrupt. Skip the znode without
+ // deleting it: an overly narrow filter pattern (e.g. "org.apache.spark.*", which
+ // does not match subpackages) must not wipe the whole recovery state on failover.
+ logError(log"Skipping persisted file ${MDC(LogKeys.FILE_NAME, filename)}, " +
+ log"rejected by the recovery serialization filter " +
+ log"(${MDC(LogKeys.CONFIG, RECOVERY_SERIALIZATION_FILTER.key)})", e)
+ None
case e: Exception =>
logWarning("Exception while reading persisted file, deleting", e)
zk.delete().forPath(workingDir + "/" + filename)
None
}
}
+
+ // Records whether the recovery serialization filter rejected anything during a read, since
+ // the JDK reports a rejection only as a generic InvalidClassException. Only this filter's
+ // rejections are recorded: a znode rejected solely by a JVM-wide jdk.serialFilter is
+ // handled like any other unreadable znode.
+ private class RecordingFilter(delegate: ObjectInputFilter) extends ObjectInputFilter {
+ var rejected = false
+
+ override def checkInput(info: ObjectInputFilter.FilterInfo): ObjectInputFilter.Status = {
+ val status = delegate.checkInput(info)
+ if (status == ObjectInputFilter.Status.REJECTED) {
+ rejected = true
+ }
+ status
+ }
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
index 1ed53868992a..715ca7c5f8d5 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/Deploy.scala
@@ -17,6 +17,7 @@
package org.apache.spark.internal.config
+import java.io.ObjectInputFilter
import java.util.Locale
import java.util.concurrent.TimeUnit
@@ -51,6 +52,26 @@ private[spark] object Deploy {
.checkValue(_ > 0, "spark.deploy.recoveryTimeout must be positive.")
.createOptional
+ val RECOVERY_SERIALIZATION_FILTER =
+ ConfigBuilder("spark.deploy.recoverySerializationFilter")
+ .doc("JEP-290 serialization filter pattern applied when the master reads back " +
+ "recovery state written by the built-in JavaSerializer (currently enforced for " +
+ "the ZOOKEEPER recovery mode). The default allows only JDK, Scala and Spark " +
+ "classes, which covers everything the master persists (ApplicationInfo, " +
+ "DriverInfo, WorkerInfo and their fields). Znodes containing any other class " +
+ "are skipped, without being deleted, during recovery instead of being " +
+ "instantiated in the newly elected master. This only hardens deserialization and " +
+ "is not a replacement for ZooKeeper ACLs, which remain the access control for the " +
+ "recovery state. The filter is applied in addition to any JVM-wide " +
+ "jdk.serialFilter; znodes rejected only by jdk.serialFilter are deleted like other " +
+ "unreadable znodes. Set to '*' to disable filtering.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .stringConf
+ .checkValue(v => v.trim.nonEmpty && ObjectInputFilter.Config.createFilter(v) != null,
+ "must be a non-empty JEP-290 filter pattern; use '*' to disable filtering.")
+ .createWithDefault("java.**;scala.**;org.apache.spark.**;!*")
+
val ZOOKEEPER_URL = ConfigBuilder("spark.deploy.zookeeper.url")
.doc(s"When `${RECOVERY_MODE.key}` is set to ZOOKEEPER, this " +
"configuration is used to set the zookeeper URL to connect to.")
diff --git a/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
index 856e639fcd9a..5dae02f3406f 100644
--- a/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/JavaSerializer.scala
@@ -61,7 +61,10 @@ private[spark] class JavaSerializationStream(
def close(): Unit = { objOut.close() }
}
-private[spark] class JavaDeserializationStream(in: InputStream, loader: ClassLoader)
+private[spark] class JavaDeserializationStream(
+ in: InputStream,
+ loader: ClassLoader,
+ filter: Option[ObjectInputFilter] = None)
extends DeserializationStream {
private val objIn = new ObjectInputStream(in) {
@@ -85,6 +88,17 @@ private[spark] class JavaDeserializationStream(in: InputStream, loader: ClassLoa
}
+ // A JEP-290 deserialization filter for callers that restrict which classes may be
+ // instantiated on read (e.g. the master recovery store). Applied per-stream so it cannot
+ // affect other JavaSerializer users. If the stream already has a filter (e.g. a JVM-wide
+ // jdk.serialFilter), merge the two instead of replacing it, so a rejection by either one
+ // rejects. The caller's filter is consulted first: merge stops at the first rejection, so
+ // this lets a caller that tracks its own rejections see all of them.
+ filter.foreach { f =>
+ objIn.setObjectInputFilter(
+ Option(objIn.getObjectInputFilter).map(ObjectInputFilter.merge(f, _)).getOrElse(f))
+ }
+
def readObject[T: ClassTag](): T = objIn.readObject().asInstanceOf[T]
def close(): Unit = { objIn.close() }
}
@@ -148,6 +162,10 @@ private[spark] class JavaSerializerInstance(
new JavaDeserializationStream(s, loader)
}
+ def deserializeStream(s: InputStream, filter: ObjectInputFilter): DeserializationStream = {
+ new JavaDeserializationStream(s, defaultClassLoader, Some(filter))
+ }
+
}
/**
diff --git a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
index 6839afdeeff8..416af7457432 100644
--- a/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/master/PersistenceEngineSuite.scala
@@ -20,12 +20,15 @@ package org.apache.spark.deploy.master
import java.net.ServerSocket
import java.nio.file.{Files, Paths}
+import java.util.Date
import java.util.concurrent.ThreadLocalRandom
+import org.apache.commons.lang3.mutable.MutableInt
import org.apache.curator.test.TestingServer
-import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite}
-import org.apache.spark.internal.config.Deploy.ZOOKEEPER_URL
+import org.apache.spark.{SecurityManager, SparkConf, SparkFunSuite, SparkIllegalArgumentException}
+import org.apache.spark.deploy.{ApplicationDescription, Command, DeployTestUtils, DriverDescription, SparkCuratorUtil}
+import org.apache.spark.internal.config.Deploy.{RECOVERY_SERIALIZATION_FILTER, ZOOKEEPER_URL}
import org.apache.spark.io.CompressionCodec
import org.apache.spark.rpc.{RpcEndpoint, RpcEnv}
import org.apache.spark.serializer.{JavaSerializer, Serializer}
@@ -132,6 +135,76 @@ class PersistenceEngineSuite extends SparkFunSuite {
}
}
+ test("SPARK-59333: ZooKeeperPersistenceEngine skips classes rejected by the filter") {
+ val conf = new SparkConf()
+ val zkTestServer = new TestingServer(findFreePort(conf))
+ try {
+ conf.set(ZOOKEEPER_URL, zkTestServer.getConnectString)
+ val engine = new ZooKeeperPersistenceEngine(conf, new JavaSerializer(conf))
+ try {
+ // A class outside the java/scala/spark allowlist is skipped on read, not
+ // instantiated.
+ engine.persist("test_filtered", new MutableInt(1))
+ assert(engine.read[AnyRef]("test_filtered").isEmpty)
+
+ // The znode is skipped, not deleted: an overly narrow filter pattern must not
+ // wipe the recovery state.
+ val zk = SparkCuratorUtil.newClient(conf)
+ try {
+ assert(zk.checkExists().forPath("/spark/master_status/test_filtered") != null)
+ } finally {
+ zk.close()
+ }
+
+ // Allowlisted JDK/Scala/Spark types still round-trip.
+ engine.persist("test_allowed", "test_allowed_value")
+ assert(engine.read[String]("test_allowed") === Seq("test_allowed_value"))
+
+ // The info classes the master actually persists must round-trip through the filter.
+ val command = new Command("", Nil, Map.empty, Nil, Nil, Nil)
+ val appDesc = new ApplicationDescription(
+ name = "test_app",
+ maxCores = None,
+ command = command,
+ appUiUrl = "",
+ defaultProfile = DeployTestUtils.defaultResourceProfile,
+ eventLogDir = None,
+ eventLogCodec = None)
+ engine.persist("test_app_info",
+ new ApplicationInfo(0, "test_app", appDesc, new Date(), null, 0))
+ assert(engine.read[ApplicationInfo]("test_app_info").map(_.id) === Seq("test_app"))
+
+ val driverDesc = new DriverDescription("", 0, 0, false, command)
+ engine.persist("test_driver_info",
+ new DriverInfo(0, "test_driver", driverDesc, new Date()))
+ assert(engine.read[DriverInfo]("test_driver_info").map(_.id) === Seq("test_driver"))
+ } finally {
+ engine.close()
+ }
+ } finally {
+ zkTestServer.stop()
+ }
+ }
+
+ test("SPARK-59333: recoverySerializationFilter rejects patterns that yield no filter") {
+ // ObjectInputFilter.Config.createFilter returns null for "" and ";", and a filter that
+ // matches nothing for blanks; either would otherwise silently disable the filtering.
+ Seq("", " ", ";").foreach { pattern =>
+ checkError(
+ exception = intercept[SparkIllegalArgumentException] {
+ new SparkConf().set(RECOVERY_SERIALIZATION_FILTER.key, pattern)
+ .get(RECOVERY_SERIALIZATION_FILTER)
+ },
+ condition = "INVALID_CONF_VALUE.REQUIREMENT",
+ parameters = Map(
+ "confName" -> RECOVERY_SERIALIZATION_FILTER.key,
+ "confValue" -> pattern,
+ "confRequirement" ->
+ "must be a non-empty JEP-290 filter pattern; use '*' to disable filtering.")
+ )
+ }
+ }
+
private def testPersistenceEngine(
conf: SparkConf, persistenceEngineCreator: Serializer => PersistenceEngine): Unit = {
val serializer = new JavaSerializer(conf)
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index ec1656b0348c..c8e6c711f8fd 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -873,6 +873,24 @@ In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spa
<td>When <code>spark.deploy.recoveryMode</code> is set to ZOOKEEPER, this configuration is used to set the zookeeper directory to store recovery state.</td>
<td>0.8.1</td>
</tr>
+ <tr>
+ <td><code>spark.deploy.recoverySerializationFilter</code></td>
+ <td>java.**;scala.**;org.apache.spark.**;!*</td>
+ <td>Serialization filter pattern applied when the master reads back recovery state that
+ the built-in JavaSerializer wrote, currently for the ZOOKEEPER recovery mode. The default
+ allows only JDK, Scala and Spark classes, which covers everything the master persists;
+ znodes containing any other class are skipped, without being deleted, during recovery
+ instead of being instantiated in the newly elected master.
+ This only hardens deserialization and is not a replacement for ZooKeeper ACLs, which
+ remain the access control for the recovery state.
+ The filter is applied in addition to any JVM-wide <code>jdk.serialFilter</code>; znodes
+ rejected only by <code>jdk.serialFilter</code> are deleted like other unreadable znodes.
+ Skipped znodes are never cleaned up by the master and are logged on every failover, so
+ operators should inspect and remove them manually.
+ Set to <code>*</code> to disable filtering.
+ </td>
+ <td>4.3.0</td>
+ </tr>
</table>
**Details**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org