You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/07/08 06:20:15 UTC
spark git commit: [SPARK-20379][CORE] Allow SSL config to reference
env variables.
Repository: spark
Updated Branches:
refs/heads/master 7896e7b99 -> 9760c15ac
[SPARK-20379][CORE] Allow SSL config to reference env variables.
This change exposes the internal code path in SparkConf that allows
configs to be read with variable substitution applied, and uses that
new method in SSLOptions so that SSL configs can reference other
variables, and more importantly, environment variables, providing
a secure way to provide passwords to Spark when using SSL.
The approach is a little bit hacky, but is the smallest change possible.
Otherwise, the concept of "namespaced configs" would have to be added
to the config system, which would create a lot of noise for not much
gain at this point.
Tested with added unit tests, and on a real cluster with SSL enabled.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #18394 from vanzin/SPARK-20379.try2.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9760c15a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9760c15a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9760c15a
Branch: refs/heads/master
Commit: 9760c15acbcf755dd5b13597ceb333576f806ecf
Parents: 7896e7b
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Sat Jul 8 14:20:09 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Sat Jul 8 14:20:09 2017 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/SSLOptions.scala | 20 ++++++++++----------
.../main/scala/org/apache/spark/SparkConf.scala | 5 +++++
.../org/apache/spark/SSLOptionsSuite.scala | 16 ++++++++++++++++
3 files changed, 31 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9760c15a/core/src/main/scala/org/apache/spark/SSLOptions.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SSLOptions.scala b/core/src/main/scala/org/apache/spark/SSLOptions.scala
index 29163e7..f86fd20 100644
--- a/core/src/main/scala/org/apache/spark/SSLOptions.scala
+++ b/core/src/main/scala/org/apache/spark/SSLOptions.scala
@@ -167,39 +167,39 @@ private[spark] object SSLOptions extends Logging {
def parse(conf: SparkConf, ns: String, defaults: Option[SSLOptions] = None): SSLOptions = {
val enabled = conf.getBoolean(s"$ns.enabled", defaultValue = defaults.exists(_.enabled))
- val port = conf.getOption(s"$ns.port").map(_.toInt)
+ val port = conf.getWithSubstitution(s"$ns.port").map(_.toInt)
port.foreach { p =>
require(p >= 0, "Port number must be a non-negative value.")
}
- val keyStore = conf.getOption(s"$ns.keyStore").map(new File(_))
+ val keyStore = conf.getWithSubstitution(s"$ns.keyStore").map(new File(_))
.orElse(defaults.flatMap(_.keyStore))
- val keyStorePassword = conf.getOption(s"$ns.keyStorePassword")
+ val keyStorePassword = conf.getWithSubstitution(s"$ns.keyStorePassword")
.orElse(defaults.flatMap(_.keyStorePassword))
- val keyPassword = conf.getOption(s"$ns.keyPassword")
+ val keyPassword = conf.getWithSubstitution(s"$ns.keyPassword")
.orElse(defaults.flatMap(_.keyPassword))
- val keyStoreType = conf.getOption(s"$ns.keyStoreType")
+ val keyStoreType = conf.getWithSubstitution(s"$ns.keyStoreType")
.orElse(defaults.flatMap(_.keyStoreType))
val needClientAuth =
conf.getBoolean(s"$ns.needClientAuth", defaultValue = defaults.exists(_.needClientAuth))
- val trustStore = conf.getOption(s"$ns.trustStore").map(new File(_))
+ val trustStore = conf.getWithSubstitution(s"$ns.trustStore").map(new File(_))
.orElse(defaults.flatMap(_.trustStore))
- val trustStorePassword = conf.getOption(s"$ns.trustStorePassword")
+ val trustStorePassword = conf.getWithSubstitution(s"$ns.trustStorePassword")
.orElse(defaults.flatMap(_.trustStorePassword))
- val trustStoreType = conf.getOption(s"$ns.trustStoreType")
+ val trustStoreType = conf.getWithSubstitution(s"$ns.trustStoreType")
.orElse(defaults.flatMap(_.trustStoreType))
- val protocol = conf.getOption(s"$ns.protocol")
+ val protocol = conf.getWithSubstitution(s"$ns.protocol")
.orElse(defaults.flatMap(_.protocol))
- val enabledAlgorithms = conf.getOption(s"$ns.enabledAlgorithms")
+ val enabledAlgorithms = conf.getWithSubstitution(s"$ns.enabledAlgorithms")
.map(_.split(",").map(_.trim).filter(_.nonEmpty).toSet)
.orElse(defaults.map(_.enabledAlgorithms))
.getOrElse(Set.empty)
http://git-wip-us.apache.org/repos/asf/spark/blob/9760c15a/core/src/main/scala/org/apache/spark/SparkConf.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index de2f475..715cfdc 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -373,6 +373,11 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging with Seria
Option(settings.get(key)).orElse(getDeprecatedConfig(key, this))
}
+ /** Get an optional value, applying variable substitution. */
+ private[spark] def getWithSubstitution(key: String): Option[String] = {
+ getOption(key).map(reader.substitute(_))
+ }
+
/** Get all parameters as a list of pairs */
def getAll: Array[(String, String)] = {
settings.entrySet().asScala.map(x => (x.getKey, x.getValue)).toArray
http://git-wip-us.apache.org/repos/asf/spark/blob/9760c15a/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala b/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala
index 6fc7cea..8eabc2b 100644
--- a/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SSLOptionsSuite.scala
@@ -22,6 +22,8 @@ import javax.net.ssl.SSLContext
import org.scalatest.BeforeAndAfterAll
+import org.apache.spark.util.SparkConfWithEnv
+
class SSLOptionsSuite extends SparkFunSuite with BeforeAndAfterAll {
test("test resolving property file as spark conf ") {
@@ -133,4 +135,18 @@ class SSLOptionsSuite extends SparkFunSuite with BeforeAndAfterAll {
assert(opts.enabledAlgorithms === Set("ABC", "DEF"))
}
+ test("variable substitution") {
+ val conf = new SparkConfWithEnv(Map(
+ "ENV1" -> "val1",
+ "ENV2" -> "val2"))
+
+ conf.set("spark.ssl.enabled", "true")
+ conf.set("spark.ssl.keyStore", "${env:ENV1}")
+ conf.set("spark.ssl.trustStore", "${env:ENV2}")
+
+ val opts = SSLOptions.parse(conf, "spark.ssl", defaults = None)
+ assert(opts.keyStore === Some(new File("val1")))
+ assert(opts.trustStore === Some(new File("val2")))
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org