You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2018/12/11 21:50:47 UTC
[spark] branch master updated: [SPARK-26239] File-based secret key
loading for SASL.
This is an automated email from the ASF dual-hosted git repository.
vanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 57d6fbf [SPARK-26239] File-based secret key loading for SASL.
57d6fbf is described below
commit 57d6fbfa8c803ce1791e7be36aba0219a1fcaa63
Author: mcheah <mc...@palantir.com>
AuthorDate: Tue Dec 11 13:49:52 2018 -0800
[SPARK-26239] File-based secret key loading for SASL.
This proposes an alternative way to load secret keys into a Spark application that is running on Kubernetes. Instead of automatically generating the secret, the secret key can reside in a file that is shared between both the driver and executor containers.
Unit tests.
Closes #23252 from mccheah/auth-secret-with-file.
Authored-by: mcheah <mc...@palantir.com>
Signed-off-by: Marcelo Vanzin <va...@cloudera.com>
---
.../scala/org/apache/spark/SecurityManager.scala | 34 ++++++++++-
.../src/main/scala/org/apache/spark/SparkEnv.scala | 4 +-
.../org/apache/spark/internal/config/package.scala | 31 ++++++++++
.../org/apache/spark/SecurityManagerSuite.scala | 66 +++++++++++++++++++++-
docs/security.md | 44 +++++++++++++++
.../k8s/features/BasicExecutorFeatureStep.scala | 16 +++---
.../features/BasicExecutorFeatureStepSuite.scala | 23 ++++++++
7 files changed, 205 insertions(+), 13 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/SecurityManager.scala b/core/src/main/scala/org/apache/spark/SecurityManager.scala
index 96e4b53..15783c9 100644
--- a/core/src/main/scala/org/apache/spark/SecurityManager.scala
+++ b/core/src/main/scala/org/apache/spark/SecurityManager.scala
@@ -17,8 +17,11 @@
package org.apache.spark
+import java.io.File
import java.net.{Authenticator, PasswordAuthentication}
import java.nio.charset.StandardCharsets.UTF_8
+import java.nio.file.Files
+import java.util.Base64
import org.apache.hadoop.io.Text
import org.apache.hadoop.security.{Credentials, UserGroupInformation}
@@ -43,7 +46,8 @@ import org.apache.spark.util.Utils
*/
private[spark] class SecurityManager(
sparkConf: SparkConf,
- val ioEncryptionKey: Option[Array[Byte]] = None)
+ val ioEncryptionKey: Option[Array[Byte]] = None,
+ authSecretFileConf: ConfigEntry[Option[String]] = AUTH_SECRET_FILE)
extends Logging with SecretKeyHolder {
import SecurityManager._
@@ -328,6 +332,7 @@ private[spark] class SecurityManager(
.orElse(Option(secretKey))
.orElse(Option(sparkConf.getenv(ENV_AUTH_SECRET)))
.orElse(sparkConf.getOption(SPARK_AUTH_SECRET_CONF))
+ .orElse(secretKeyFromFile())
.getOrElse {
throw new IllegalArgumentException(
s"A secret key must be specified via the $SPARK_AUTH_SECRET_CONF config")
@@ -348,7 +353,6 @@ private[spark] class SecurityManager(
*/
def initializeAuth(): Unit = {
import SparkMasterRegex._
- val k8sRegex = "k8s.*".r
if (!sparkConf.get(NETWORK_AUTH_ENABLED)) {
return
@@ -371,7 +375,14 @@ private[spark] class SecurityManager(
return
}
- secretKey = Utils.createSecret(sparkConf)
+ if (sparkConf.get(AUTH_SECRET_FILE_DRIVER).isDefined !=
+ sparkConf.get(AUTH_SECRET_FILE_EXECUTOR).isDefined) {
+ throw new IllegalArgumentException(
+ "Invalid secret configuration: Secret files must be specified for both the driver and the" +
+ " executors, not only one or the other.")
+ }
+
+ secretKey = secretKeyFromFile().getOrElse(Utils.createSecret(sparkConf))
if (storeInUgi) {
val creds = new Credentials()
@@ -380,6 +391,22 @@ private[spark] class SecurityManager(
}
}
+ private def secretKeyFromFile(): Option[String] = {
+ sparkConf.get(authSecretFileConf).flatMap { secretFilePath =>
+ sparkConf.getOption(SparkLauncher.SPARK_MASTER).map {
+ case k8sRegex() =>
+ val secretFile = new File(secretFilePath)
+ require(secretFile.isFile, s"No file found containing the secret key at $secretFilePath.")
+ val base64Key = Base64.getEncoder.encodeToString(Files.readAllBytes(secretFile.toPath))
+ require(!base64Key.isEmpty, s"Secret key from file located at $secretFilePath is empty.")
+ base64Key
+ case _ =>
+ throw new IllegalArgumentException(
+ "Secret keys provided via files is only allowed in Kubernetes mode.")
+ }
+ }
+ }
+
// Default SecurityManager only has a single secret key, so ignore appId.
override def getSaslUser(appId: String): String = getSaslUser()
override def getSecretKey(appId: String): String = getSecretKey()
@@ -387,6 +414,7 @@ private[spark] class SecurityManager(
private[spark] object SecurityManager {
+ val k8sRegex = "k8s.*".r
val SPARK_AUTH_CONF = NETWORK_AUTH_ENABLED.key
val SPARK_AUTH_SECRET_CONF = "spark.authenticate.secret"
// This is used to set auth secret to an executor's env variable. It should have the same
diff --git a/core/src/main/scala/org/apache/spark/SparkEnv.scala b/core/src/main/scala/org/apache/spark/SparkEnv.scala
index 66038ee..de0c857 100644
--- a/core/src/main/scala/org/apache/spark/SparkEnv.scala
+++ b/core/src/main/scala/org/apache/spark/SparkEnv.scala
@@ -232,8 +232,8 @@ object SparkEnv extends Logging {
if (isDriver) {
assert(listenerBus != null, "Attempted to create driver SparkEnv with null listener bus!")
}
-
- val securityManager = new SecurityManager(conf, ioEncryptionKey)
+ val authSecretFileConf = if (isDriver) AUTH_SECRET_FILE_DRIVER else AUTH_SECRET_FILE_EXECUTOR
+ val securityManager = new SecurityManager(conf, ioEncryptionKey, authSecretFileConf)
if (isDriver) {
securityManager.initializeAuth()
}
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index 85bb557..f1c1c03 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -424,6 +424,37 @@ package object config {
.booleanConf
.createWithDefault(false)
+ private[spark] val AUTH_SECRET_FILE =
+ ConfigBuilder("spark.authenticate.secret.file")
+ .doc("Path to a file that contains the authentication secret to use. The secret key is " +
+ "loaded from this path on both the driver and the executors if overrides are not set for " +
+ "either entity (see below). File-based secret keys are only allowed when using " +
+ "Kubernetes.")
+ .stringConf
+ .createOptional
+
+ private[spark] val AUTH_SECRET_FILE_DRIVER =
+ ConfigBuilder("spark.authenticate.secret.driver.file")
+ .doc("Path to a file that contains the authentication secret to use. Loaded by the " +
+ "driver. In Kubernetes client mode it is often useful to set a different secret " +
+ "path for the driver vs. the executors, since the driver may not be running in " +
+ "a pod unlike the executors. If this is set, an accompanying secret file must " +
+ "be specified for the executors. The fallback configuration allows the same path to be " +
+ "used for both the driver and the executors when running in cluster mode. File-based " +
+ "secret keys are only allowed when using Kubernetes.")
+ .fallbackConf(AUTH_SECRET_FILE)
+
+ private[spark] val AUTH_SECRET_FILE_EXECUTOR =
+ ConfigBuilder("spark.authenticate.secret.executor.file")
+ .doc("Path to a file that contains the authentication secret to use. Loaded by the " +
+ "executors only. In Kubernetes client mode it is often useful to set a different " +
+ "secret path for the driver vs. the executors, since the driver may not be running " +
+ "in a pod unlike the executors. If this is set, an accompanying secret file must be " +
+ "specified for the executors. The fallback configuration allows the same path to be " +
+ "used for both the driver and the executors when running in cluster mode. File-based " +
+ "secret keys are only allowed when using Kubernetes.")
+ .fallbackConf(AUTH_SECRET_FILE)
+
private[spark] val NETWORK_ENCRYPTION_ENABLED =
ConfigBuilder("spark.network.crypto.enabled")
.booleanConf
diff --git a/core/src/test/scala/org/apache/spark/SecurityManagerSuite.scala b/core/src/test/scala/org/apache/spark/SecurityManagerSuite.scala
index eec8004..e9061f4 100644
--- a/core/src/test/scala/org/apache/spark/SecurityManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/SecurityManagerSuite.scala
@@ -19,7 +19,9 @@ package org.apache.spark
import java.io.File
import java.nio.charset.StandardCharsets.UTF_8
+import java.nio.file.Files
import java.security.PrivilegedExceptionAction
+import java.util.Base64
import org.apache.hadoop.security.UserGroupInformation
@@ -395,9 +397,54 @@ class SecurityManagerSuite extends SparkFunSuite with ResetSystemProperties {
assert(keyFromEnv === new SecurityManager(conf2).getSecretKey())
}
+ test("use executor-specific secret file configuration.") {
+ val secretFileFromDriver = createTempSecretFile("driver-secret")
+ val secretFileFromExecutor = createTempSecretFile("executor-secret")
+ val conf = new SparkConf()
+ .setMaster("k8s://127.0.0.1")
+ .set(AUTH_SECRET_FILE_DRIVER, Some(secretFileFromDriver.getAbsolutePath))
+ .set(AUTH_SECRET_FILE_EXECUTOR, Some(secretFileFromExecutor.getAbsolutePath))
+ .set(SecurityManager.SPARK_AUTH_CONF, "true")
+ val mgr = new SecurityManager(conf, authSecretFileConf = AUTH_SECRET_FILE_EXECUTOR)
+ assert(encodeFileAsBase64(secretFileFromExecutor) === mgr.getSecretKey())
+ }
+
+ test("secret file must be defined in both driver and executor") {
+ val conf1 = new SparkConf()
+ .set(AUTH_SECRET_FILE_DRIVER, Some("/tmp/driver-secret.txt"))
+ .set(SecurityManager.SPARK_AUTH_CONF, "true")
+ val mgr1 = new SecurityManager(conf1)
+ intercept[IllegalArgumentException] {
+ mgr1.initializeAuth()
+ }
+
+ val conf2 = new SparkConf()
+ .set(AUTH_SECRET_FILE_EXECUTOR, Some("/tmp/executor-secret.txt"))
+ .set(SecurityManager.SPARK_AUTH_CONF, "true")
+ val mgr2 = new SecurityManager(conf2)
+ intercept[IllegalArgumentException] {
+ mgr2.initializeAuth()
+ }
+ }
+
+ Seq("yarn", "local", "local[*]", "local[1,2]", "mesos://localhost:8080").foreach { master =>
+ test(s"master $master cannot use file mounted secrets") {
+ val conf = new SparkConf()
+ .set(AUTH_SECRET_FILE, "/tmp/secret.txt")
+ .set(SecurityManager.SPARK_AUTH_CONF, "true")
+ .setMaster(master)
+ intercept[IllegalArgumentException] {
+ new SecurityManager(conf).getSecretKey()
+ }
+ intercept[IllegalArgumentException] {
+ new SecurityManager(conf).initializeAuth()
+ }
+ }
+ }
+
// How is the secret expected to be generated and stored.
object SecretTestType extends Enumeration {
- val MANUAL, AUTO, UGI = Value
+ val MANUAL, AUTO, UGI, FILE = Value
}
import SecretTestType._
@@ -408,6 +455,7 @@ class SecurityManagerSuite extends SparkFunSuite with ResetSystemProperties {
("local[*]", UGI),
("local[1, 2]", UGI),
("k8s://127.0.0.1", AUTO),
+ ("k8s://127.0.1.1", FILE),
("local-cluster[2, 1, 1024]", MANUAL),
("invalid", MANUAL)
).foreach { case (master, secretType) =>
@@ -440,6 +488,12 @@ class SecurityManagerSuite extends SparkFunSuite with ResetSystemProperties {
intercept[IllegalArgumentException] {
mgr.getSecretKey()
}
+
+ case FILE =>
+ val secretFile = createTempSecretFile()
+ conf.set(AUTH_SECRET_FILE, secretFile.getAbsolutePath)
+ mgr.initializeAuth()
+ assert(encodeFileAsBase64(secretFile) === mgr.getSecretKey())
}
}
}
@@ -447,5 +501,15 @@ class SecurityManagerSuite extends SparkFunSuite with ResetSystemProperties {
}
}
+ private def encodeFileAsBase64(secretFile: File) = {
+ Base64.getEncoder.encodeToString(Files.readAllBytes(secretFile.toPath))
+ }
+
+ private def createTempSecretFile(contents: String = "test-secret"): File = {
+ val secretDir = Utils.createTempDir("temp-secrets")
+ val secretFile = new File(secretDir, "temp-secret.txt")
+ Files.write(secretFile.toPath, contents.getBytes(UTF_8))
+ secretFile
+ }
}
diff --git a/docs/security.md b/docs/security.md
index 2a4f3c0..8416ed9 100644
--- a/docs/security.md
+++ b/docs/security.md
@@ -66,6 +66,50 @@ Kubernetes admin to ensure that Spark authentication is secure.
</tr>
</table>
+Alternatively, one can mount authentication secrets using files and Kubernetes secrets that
+the user mounts into their pods.
+
+<table class="table">
+<tr><th>Property Name</th><th>Default</th><th>Meaning</th></tr>
+<tr>
+ <td><code>spark.authenticate.secret.file</code></td>
+ <td>None</td>
+ <td>
+ Path pointing to the secret key to use for securing connections. Ensure that the
+ contents of the file have been securely generated. This file is loaded on both the driver
+ and the executors unless other settings override this (see below).
+ </td>
+</tr>
+<tr>
+ <td><code>spark.authenticate.secret.driver.file</code></td>
+ <td>The value of <code>spark.authenticate.secret.file</code></td>
+ <td>
+ When specified, overrides the location that the Spark driver reads to load the secret.
+ Useful when in client mode, when the location of the secret file may differ in the pod versus
+ the node the driver is running in. When this is specified,
+ <code>spark.authenticate.secret.executor.file</code> must be specified so that the driver
+ and the executors can both use files to load the secret key. Ensure that the contents of the file
+ on the driver is identical to the contents of the file on the executors.
+ </td>
+</tr>
+<tr>
+ <td><code>spark.authenticate.secret.executor.file</code></td>
+ <td>The value of <code>spark.authenticate.secret.file</code></td>
+ <td>
+ When specified, overrides the location that the Spark executors read to load the secret.
+ Useful in client mode, when the location of the secret file may differ in the pod versus
+ the node the driver is running in. When this is specified,
+ <code>spark.authenticate.secret.driver.file</code> must be specified so that the driver
+ and the executors can both use files to load the secret key. Ensure that the contents of the file
+ on the driver is identical to the contents of the file on the executors.
+ </td>
+</tr>
+</table>
+
+Note that when using files, Spark will not mount these files into the containers for you. It is up
+you to ensure that the secret files are deployed securely into your containers and that the driver's
+secret file agrees with the executors' secret file.
+
## Encryption
Spark supports AES-based encryption for RPC connections. For encryption to be enabled, RPC
diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
index 939aa88..4bcf4c9 100644
--- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
+++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStep.scala
@@ -24,7 +24,7 @@ import org.apache.spark.{SecurityManager, SparkConf, SparkException}
import org.apache.spark.deploy.k8s._
import org.apache.spark.deploy.k8s.Config._
import org.apache.spark.deploy.k8s.Constants._
-import org.apache.spark.internal.config.{EXECUTOR_CLASS_PATH, EXECUTOR_JAVA_OPTIONS, EXECUTOR_MEMORY, EXECUTOR_MEMORY_OVERHEAD, PYSPARK_EXECUTOR_MEMORY}
+import org.apache.spark.internal.config._
import org.apache.spark.rpc.RpcEndpointAddress
import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend
import org.apache.spark.util.Utils
@@ -112,12 +112,14 @@ private[spark] class BasicExecutorFeatureStep(
.build())
.build())
} ++ {
- Option(secMgr.getSecretKey()).map { authSecret =>
- new EnvVarBuilder()
- .withName(SecurityManager.ENV_AUTH_SECRET)
- .withValue(authSecret)
- .build()
- }
+ if (kubernetesConf.get(AUTH_SECRET_FILE_EXECUTOR).isEmpty) {
+ Option(secMgr.getSecretKey()).map { authSecret =>
+ new EnvVarBuilder()
+ .withName(SecurityManager.ENV_AUTH_SECRET)
+ .withValue(authSecret)
+ .build()
+ }
+ } else None
} ++ {
kubernetesConf.get(EXECUTOR_CLASS_PATH).map { cp =>
new EnvVarBuilder()
diff --git a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
index 6aa8626..05989d9 100644
--- a/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
+++ b/resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/features/BasicExecutorFeatureStepSuite.scala
@@ -16,6 +16,10 @@
*/
package org.apache.spark.deploy.k8s.features
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+
import scala.collection.JavaConverters._
import io.fabric8.kubernetes.api.model._
@@ -158,6 +162,25 @@ class BasicExecutorFeatureStepSuite extends SparkFunSuite with BeforeAndAfter {
checkEnv(executor, conf, Map(SecurityManager.ENV_AUTH_SECRET -> secMgr.getSecretKey()))
}
+ test("Auth secret shouldn't propagate if files are loaded.") {
+ val secretDir = Utils.createTempDir("temp-secret")
+ val secretFile = new File(secretDir, "secret-file.txt")
+ Files.write(secretFile.toPath, "some-secret".getBytes(StandardCharsets.UTF_8))
+ val conf = baseConf.clone()
+ .set(NETWORK_AUTH_ENABLED, true)
+ .set(AUTH_SECRET_FILE, secretFile.getAbsolutePath)
+ .set("spark.master", "k8s://127.0.0.1")
+ val secMgr = new SecurityManager(conf)
+ secMgr.initializeAuth()
+
+ val step = new BasicExecutorFeatureStep(KubernetesTestConf.createExecutorConf(sparkConf = conf),
+ secMgr)
+
+ val executor = step.configurePod(SparkPod.initialPod())
+ assert(!KubernetesFeaturesTestUtils.containerHasEnvVar(
+ executor.container, SecurityManager.ENV_AUTH_SECRET))
+ }
+
// There is always exactly one controller reference, and it points to the driver pod.
private def checkOwnerReferences(executor: Pod, driverPodUid: String): Unit = {
assert(executor.getMetadata.getOwnerReferences.size() === 1)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org