You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ho...@apache.org on 2026/09/02 23:52:31 UTC
(spark) branch branch-3.5 updated: [CORE][MINOR] Expand default redaction regex
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new e14a77481a8f [CORE][MINOR] Expand default redaction regex
e14a77481a8f is described below
commit e14a77481a8f22bb566863958ffb64423d94896e
Author: Holden Karau <ho...@snowflake.com>
AuthorDate: Tue Sep 1 19:03:37 2026 +0000
[CORE][MINOR] Expand default redaction regex
Co-authored-by: Cursor <cu...@cursor.com>
Co-authored-by: Holden Karau <ho...@pigscanfly.ca>
---
core/src/main/scala/org/apache/spark/internal/config/package.scala | 2 +-
core/src/test/scala/org/apache/spark/util/UtilsSuite.scala | 3 ++-
docs/configuration.md | 2 +-
.../src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java | 3 ++-
.../test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java | 2 ++
5 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 938d6ec2e01b..98dc18f2f4ce 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -1155,7 +1155,7 @@ package object config {
"like YARN and event logs.")
.version("2.1.2")
.regexConf
- .createWithDefault("(?i)secret|password|token|access[.]?key".r)
+ .createWithDefault("(?i)secret|password|token|access[.]?key|credential".r)
private[spark] val STRING_REDACTION_PATTERN =
ConfigBuilder("spark.redaction.string.regex")
diff --git a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
index c09ea80bb9fe..74d8d765c528 100644
--- a/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/UtilsSuite.scala
@@ -1098,7 +1098,8 @@ class UtilsSuite extends SparkFunSuite with ResetSystemProperties {
"spark.hadoop.fs.s3.awsAccessKeyId",
"spark.hadoop.fs.s3a.access.key",
"spark.my.password",
- "spark.my.sECreT")
+ "spark.my.sECreT",
+ "spark.sql.catalog.mycatalog.credential")
secretKeys.foreach { key => sparkConf.set(key, "sensitive_value") }
// Set a non-secret key
sparkConf.set("spark.regular.property", "regular_value")
diff --git a/docs/configuration.md b/docs/configuration.md
index f3b29254283b..617fef3be234 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -748,7 +748,7 @@ Apart from these, the following properties are also available, and may be useful
</tr>
<tr>
<td><code>spark.redaction.regex</code></td>
- <td>(?i)secret|password|token|access[.]key</td>
+ <td>(?i)secret|password|token|access[.]?key|credential</td>
<td>
Regex to decide which Spark configuration properties and environment variables in driver and
executor environments contain sensitive information. When this regex matches a property key or
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index ff729cd1cb6d..c9dcfb5a4f31 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -35,7 +35,8 @@ class CommandBuilderUtils {
static final String ENV_SPARK_HOME = "SPARK_HOME";
// This should be consistent with org.apache.spark.internal.config.SECRET_REDACTION_PATTERN
// We maintain this copy to avoid depending on `core` module.
- static final String SECRET_REDACTION_PATTERN = "(?i)secret|password|token|access[.]?key";
+ static final String SECRET_REDACTION_PATTERN =
+ "(?i)secret|password|token|access[.]?key|credential";
static final Pattern redactPattern = Pattern.compile(SECRET_REDACTION_PATTERN);
static final Pattern keyValuePattern = Pattern.compile("-D(.+?)=(.+)");
diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
index 1b2c683880c2..df5695b04d8c 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
@@ -74,6 +74,8 @@ public class CommandBuilderUtilsSuite {
assertEquals(redact("-Dk=v"), "-Dk=v");
assertEquals(redact("-Dk=secret"), "-Dk=secret");
assertEquals(redact("-DsecretKey=my-secret"), "-DsecretKey=*********(redacted)");
+ assertEquals(redact("-Dspark.sql.catalog.foo.credential=my-credential"),
+ "-Dspark.sql.catalog.foo.credential=*********(redacted)");
assertEquals(redactCommandLineArgs(Arrays.asList("-DsecretKey=my-secret")),
Arrays.asList("-DsecretKey=*********(redacted)"));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org