[spark] branch branch-2.4 updated: [SPARK-27244][CORE] Redact Passwords While Using Option logConf=true

[sr...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

srowen pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-2.4 by this push:
     new 4edc535  [SPARK-27244][CORE] Redact Passwords While Using Option logConf=true
4edc535 is described below

commit 4edc535893b2694ed4b2569b04a96961b3036c6a
Author: Ninad Ingole <ro...@example.com>
AuthorDate: Fri Mar 29 14:16:53 2019 -0500

    [SPARK-27244][CORE] Redact Passwords While Using Option logConf=true
    
    ## What changes were proposed in this pull request?
    
    When logConf is set to true, config keys that contain password were printed in cleartext in driver log. This change uses the already present redact method in Utils, to redact all the passwords based on redact pattern in SparkConf and then print the conf to driver log thus ensuring that sensitive information like passwords is not printed in clear text.
    
    ## How was this patch tested?
    This patch was tested through `SparkConfSuite` & then entire unit test through sbt
    
    Please review http://spark.apache.org/contributing.html before opening a pull request.
    
    Closes #24196 from ninadingole/SPARK-27244.
    
    Authored-by: Ninad Ingole <ro...@example.com>
    Signed-off-by: Sean Owen <se...@databricks.com>
    (cherry picked from commit dbc7ce18b934fbfd0743b1348fc1265778f07027)
    Signed-off-by: Sean Owen <se...@databricks.com>
---
 core/src/main/scala/org/apache/spark/SparkConf.scala      |  2 +-
 core/src/test/scala/org/apache/spark/SparkConfSuite.scala | 10 +++++++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 854dfbd..ccafe16 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -625,7 +625,7 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
    * configuration out for debugging.
    */
   def toDebugString: String = {
-    getAll.sorted.map{case (k, v) => k + "=" + v}.mkString("\n")
+    Utils.redact(this, getAll).sorted.map { case (k, v) => k + "=" + v }.mkString("\n")
   }
 
 }
diff --git a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
index 0d06b02..21424bd 100644
--- a/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SparkConfSuite.scala
@@ -30,7 +30,7 @@ import org.apache.spark.deploy.history.config._
 import org.apache.spark.internal.config._
 import org.apache.spark.network.util.ByteUnit
 import org.apache.spark.serializer.{JavaSerializer, KryoRegistrator, KryoSerializer}
-import org.apache.spark.util.{ResetSystemProperties, RpcUtils}
+import org.apache.spark.util.{ResetSystemProperties, RpcUtils, Utils}
 
 class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSystemProperties {
   test("Test byteString conversion") {
@@ -339,6 +339,14 @@ class SparkConfSuite extends SparkFunSuite with LocalSparkContext with ResetSyst
     }
   }
 
+  test("SPARK-27244 toDebugString should redact passwords") {
+    val conf = new SparkConf().set("dummy.password", "dummy-password")
+    conf.validateSettings()
+
+    assert(conf.get("dummy.password") === "dummy-password")
+    assert(conf.toDebugString.contains(s"dummy.password=${Utils.REDACTION_REPLACEMENT_TEXT}"))
+  }
+
   val defaultIllegalValue = "SomeIllegalValue"
   val illegalValueTests : Map[String, (SparkConf, String) => Any] = Map(
     "getTimeAsSeconds" -> (_.getTimeAsSeconds(_)),


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org