You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2017/09/01 17:29:42 UTC
spark git commit: [SPARK-21728][CORE] Follow up: fix user config,
auth in SparkSubmit logging.
Repository: spark
Updated Branches:
refs/heads/master 648a8626b -> 0bdbefe9d
[SPARK-21728][CORE] Follow up: fix user config, auth in SparkSubmit logging.
- SecurityManager complains when auth is enabled but no secret is defined;
SparkSubmit doesn't use the auth functionality of the SecurityManager,
so use a dummy secret to work around the exception.
- Only reset the log4j configuration when Spark was the one initializing
it, otherwise user-defined log configuration may be lost.
Tested with the log config file posted to the bug, on a secured YARN cluster.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #19089 from vanzin/SPARK-21728.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0bdbefe9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0bdbefe9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0bdbefe9
Branch: refs/heads/master
Commit: 0bdbefe9dd1e7c95c58ea6b52d3b264794abbc0e
Parents: 648a862
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Fri Sep 1 10:29:36 2017 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Fri Sep 1 10:29:36 2017 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 3 +++
core/src/main/scala/org/apache/spark/internal/Logging.scala | 9 ++++++++-
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0bdbefe9/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 38604fe..ea9c9bd 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -352,6 +352,9 @@ object SparkSubmit extends CommandLineUtils with Logging {
var localJars: String = null
var localPyFiles: String = null
if (deployMode == CLIENT) {
+ // This security manager will not need an auth secret, but set a dummy value in case
+ // spark.authenticate is enabled, otherwise an exception is thrown.
+ sparkConf.set(SecurityManager.SPARK_AUTH_SECRET_CONF, "unused")
val secMgr = new SecurityManager(sparkConf)
localPrimaryResource = Option(args.primaryResource).map {
downloadFile(_, targetDir, sparkConf, hadoopConf, secMgr)
http://git-wip-us.apache.org/repos/asf/spark/blob/0bdbefe9/core/src/main/scala/org/apache/spark/internal/Logging.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/internal/Logging.scala b/core/src/main/scala/org/apache/spark/internal/Logging.scala
index cea9964..c0d709a 100644
--- a/core/src/main/scala/org/apache/spark/internal/Logging.scala
+++ b/core/src/main/scala/org/apache/spark/internal/Logging.scala
@@ -120,6 +120,7 @@ trait Logging {
val log4j12Initialized = LogManager.getRootLogger.getAllAppenders.hasMoreElements
// scalastyle:off println
if (!log4j12Initialized) {
+ Logging.defaultSparkLog4jConfig = true
val defaultLogProps = "org/apache/spark/log4j-defaults.properties"
Option(Utils.getSparkClassLoader.getResource(defaultLogProps)) match {
case Some(url) =>
@@ -164,6 +165,7 @@ trait Logging {
private[spark] object Logging {
@volatile private var initialized = false
@volatile private var defaultRootLevel: Level = null
+ @volatile private var defaultSparkLog4jConfig = false
val initLock = new Object()
try {
@@ -186,7 +188,12 @@ private[spark] object Logging {
*/
def uninitialize(): Unit = initLock.synchronized {
if (isLog4j12()) {
- LogManager.resetConfiguration()
+ if (defaultSparkLog4jConfig) {
+ defaultSparkLog4jConfig = false
+ LogManager.resetConfiguration()
+ } else {
+ LogManager.getRootLogger().setLevel(defaultRootLevel)
+ }
}
this.initialized = false
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org