You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/08/06 06:23:45 UTC
(spark) branch branch-4.3 updated: [SPARK-58531][SQL][CONNECT] Make spark.sql.artifact.copyFromLocalToFs.allowDestLocal a static conf
This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new 950c049481ee [SPARK-58531][SQL][CONNECT] Make spark.sql.artifact.copyFromLocalToFs.allowDestLocal a static conf
950c049481ee is described below
commit 950c049481ee9a250a68348a68d1a6b8f7219cb0
Author: haoyangeng-db <ha...@gmail.com>
AuthorDate: Thu Aug 6 14:23:00 2026 +0800
[SPARK-58531][SQL][CONNECT] Make spark.sql.artifact.copyFromLocalToFs.allowDestLocal a static conf
### What changes were proposed in this pull request?
Completes the Spark 4.0 migration of the `copyFromLocalToFs` local-destination gate: `spark.sql.artifact.copyFromLocalToFs.allowDestLocal` becomes a static SQL conf and the deprecated `spark.connect.copyFromLocalToFs.allowDestLocal` is removed, leaving one conf for the gate.
### Why are the changes needed?
The config controls whether a caller can write to an arbitrary path on the driver's local filesystem, so it is meant to be set by whoever starts the driver -- as its own doc says, "when starting spark driver". The deprecated Connect conf was a static conf and enforced that, but its replacement was declared as a runtime conf, so a session could set it and override the value the driver was started with. Declaring the replacement in StaticSQLConf restores that property.
### Does this PR introduce _any_ user-facing change?
N/A
### How was this patch tested?
New and existing tests.
### Was this patch authored or co-authored using generative AI tooling?
Co-authored by Claude Code
Closes #57730 from haoyangeng-db/sec-23675-remove-artifact-allowdestlocal-conf.
Authored-by: haoyangeng-db <ha...@gmail.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit fc409d369eb280597f08e95c31d1f5dd654d6065)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../org/apache/spark/sql/internal/SQLConf.scala | 19 +-----------
.../apache/spark/sql/internal/StaticSQLConf.scala | 12 ++++++++
.../apache/spark/sql/connect/config/Connect.scala | 12 --------
.../spark/sql/artifact/ArtifactManager.scala | 10 +++----
.../spark/sql/artifact/ArtifactManagerSuite.scala | 35 ++++++++++++++++++++--
.../configs-without-binding-policy-exceptions | 1 -
6 files changed, 50 insertions(+), 39 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 8a0d88346b63..92f53c3ad7ad 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -7728,23 +7728,6 @@ object SQLConf {
.booleanConf
.createWithDefault(true)
- // Deprecate "spark.connect.copyFromLocalToFs.allowDestLocal" in favor of this config. This is
- // currently optional because we don't want to break existing users who are using the old config.
- // If this config is set, then we override the deprecated config.
- val ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL =
- buildConf("spark.sql.artifact.copyFromLocalToFs.allowDestLocal")
- .internal()
- .doc("""
- |Allow `spark.copyFromLocalToFs` destination to be local file system
- | path on spark driver node when
- |`spark.sql.artifact.copyFromLocalToFs.allowDestLocal` is true.
- |This will allow user to overwrite arbitrary file on spark
- |driver node we should only enable it for testing purpose.
- |""".stripMargin)
- .version("4.0.0")
- .booleanConf
- .createOptional
-
val LEGACY_RETAIN_FRACTION_DIGITS_FIRST =
buildConf("spark.sql.legacy.decimal.retainFractionDigitsOnTruncate")
.internal()
@@ -8299,7 +8282,7 @@ object SQLConf {
DeprecatedConfig(ESCAPED_STRING_LITERALS.key, "4.0",
"Use raw string literals with the `r` prefix instead. "),
DeprecatedConfig("spark.connect.copyFromLocalToFs.allowDestLocal", "4.0",
- s"Use '${ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key}' instead."),
+ "Use 'spark.sql.artifact.copyFromLocalToFs.allowDestLocal' instead."),
DeprecatedConfig(ALLOW_ZERO_INDEX_IN_FORMAT_STRING.key, "4.0", "Increase indexes by 1 " +
"in `strfmt` of the `format_string` function. Refer to the first argument by \"1$\"."),
DeprecatedConfig(SHUFFLE_DEPENDENCY_FILE_CLEANUP_ENABLED.key, "4.1",
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
index 72e36250b068..8b2cfc8a8d02 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/StaticSQLConf.scala
@@ -338,6 +338,18 @@ object StaticSQLConf {
.booleanConf
.createWithDefault(true)
+ val ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL =
+ buildStaticConf("spark.sql.artifact.copyFromLocalToFs.allowDestLocal")
+ .internal()
+ .doc("Allow the `copyFromLocalToFs` destination to be a local file system path on the " +
+ "driver node. This lets the caller overwrite arbitrary files on the driver node, so it " +
+ "should only be enabled for testing purposes. This is a static conf: it can only be set " +
+ "when starting the driver, and not from a session.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .booleanConf
+ .createWithDefault(false)
+
val REFLECT_ALLOW_LIST = buildStaticConf("spark.sql.reflect.allowList")
.doc("A comma-separated allow list of regular expressions matched against the canonical " +
"static method name (in the form `class.method`, e.g. `java.util.UUID.randomUUID`) " +
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
index f5a71a667465..10531e063224 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/config/Connect.scala
@@ -22,7 +22,6 @@ import java.util.concurrent.TimeUnit
import org.apache.spark.SparkEnv
import org.apache.spark.network.util.ByteUnit
import org.apache.spark.sql.connect.common.config.ConnectCommon
-import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.internal.SQLConf.buildConf
object Connect {
@@ -302,17 +301,6 @@ object Connect {
.intConf
.createWithDefault(200)
- val CONNECT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL =
- buildStaticConf("spark.connect.copyFromLocalToFs.allowDestLocal")
- .internal()
- .doc(s"""
- |(Deprecated since Spark 4.0, please set
- |'${SQLConf.ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key}' instead.
- |""".stripMargin)
- .version("3.5.0")
- .booleanConf
- .createWithDefault(false)
-
val CONNECT_UI_SESSION_LIMIT = buildStaticConf("spark.sql.connect.ui.retainedSessions")
.doc("The number of client sessions kept in the Spark Connect UI history.")
.version("3.5.0")
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
index 804b5269c929..9b22cf4d9dd6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/artifact/ArtifactManager.scala
@@ -36,7 +36,7 @@ import org.apache.spark.internal.{Logging, LogKeys}
import org.apache.spark.internal.config.{CONNECT_SCALA_UDF_STUB_PREFIXES, EXECUTOR_USER_CLASS_PATH_FIRST}
import org.apache.spark.sql.Artifact
import org.apache.spark.sql.classic.SparkSession
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.util.ArtifactUtils
import org.apache.spark.storage.{BlockManager, CacheId, StorageLevel}
import org.apache.spark.util.{ChildFirstURLClassLoader, StubClassLoader, Utils}
@@ -504,10 +504,8 @@ class ArtifactManager(session: SparkSession) extends AutoCloseable with Logging
val localPath = serverLocalStagingPath
val fs = destFSPath.getFileSystem(hadoopConf)
if (fs.isInstanceOf[LocalFileSystem]) {
- val allowDestLocalConf =
- session.sessionState.conf.getConf(SQLConf.ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL)
- .getOrElse(
- session.conf.get("spark.connect.copyFromLocalToFs.allowDestLocal").contains("true"))
+ val allowDestLocalConf = session.sessionState.conf.getConf(
+ StaticSQLConf.ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL)
if (!allowDestLocalConf) {
// To avoid security issue, by default,
@@ -517,7 +515,7 @@ class ArtifactManager(session: SparkSession) extends AutoCloseable with Logging
// We can temporarily allow the behavior by setting spark config
// `spark.sql.artifact.copyFromLocalToFs.allowDestLocal`
// to `true` when starting spark driver, we should only enable it for testing
- // purpose.
+ // purpose. It is a static conf, so it cannot be set from a session.
throw new SparkUnsupportedOperationException("_LEGACY_ERROR_TEMP_3161")
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
index a1678335b775..0391103aa283 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/artifact/ArtifactManagerSuite.scala
@@ -22,10 +22,10 @@ import java.nio.file.{Files, Path, Paths}
import org.apache.spark.{SparkConf, SparkException, SparkRuntimeException}
import org.apache.spark.metrics.source.CodegenMetrics
-import org.apache.spark.sql.Artifact
+import org.apache.spark.sql.{AnalysisException, Artifact}
import org.apache.spark.sql.classic.SparkSession
import org.apache.spark.sql.functions.col
-import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf}
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.DataTypes
import org.apache.spark.storage.CacheId
@@ -253,6 +253,30 @@ class ArtifactManagerSuite extends SharedSparkSession {
assert(copiedClassFile.exists())
}
+ test("SPARK-58531: allowDestLocal cannot be set from a session") {
+ // The conf gates writes to a local filesystem destination on the driver, so it must stay a
+ // static conf: a session that could turn it on would be able to write to arbitrary paths on
+ // the driver. Guard against it being made session-settable again.
+ val key = StaticSQLConf.ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key
+ assert(SQLConf.isStaticConfigKey(key))
+ checkError(
+ exception = intercept[AnalysisException](spark.conf.set(key, "true")),
+ condition = "CANNOT_MODIFY_STATIC_CONFIG",
+ parameters = Map("key" -> s""""$key""""))
+ }
+
+ test("SPARK-58531: StaticSQLConf can initialize before SQLConf") {
+ // Use a fresh process because both objects may already be initialized in this test JVM.
+ val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
+ val process = Utils.executeCommand(
+ Seq(
+ s"$sparkHome/bin/spark-class",
+ StaticSQLConfInitializationTestApp.getClass.getCanonicalName.stripSuffix("$")),
+ new File(sparkHome),
+ Map("SPARK_TESTING" -> "1", "SPARK_HOME" -> sparkHome))
+ assert(process.waitFor() === 0)
+ }
+
test("Removal of resources") {
withTempPath { path =>
@@ -714,3 +738,10 @@ class ArtifactManagerSuite extends SharedSparkSession {
}
}
}
+
+object StaticSQLConfInitializationTestApp {
+ def main(args: Array[String]): Unit = {
+ val key = StaticSQLConf.ARTIFACT_COPY_FROM_LOCAL_TO_FS_ALLOW_DEST_LOCAL.key
+ require(key == "spark.sql.artifact.copyFromLocalToFs.allowDestLocal")
+ }
+}
diff --git a/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions b/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions
index 37598f184c02..69f1adf5f62a 100644
--- a/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions
+++ b/sql/hive/src/test/resources/conf/binding-policy-exceptions/configs-without-binding-policy-exceptions
@@ -453,7 +453,6 @@ spark.sql.ansi.enabled
spark.sql.ansi.enforceReservedKeywords
spark.sql.ansi.relationPrecedence
spark.sql.artifact.cacheStorageLevel
-spark.sql.artifact.copyFromLocalToFs.allowDestLocal
spark.sql.artifact.isolation.alwaysApplyClassloader
spark.sql.artifact.isolation.enabled
spark.sql.assumeAnsiFalseIfNotPersisted.enabled
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org