You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/07/07 00:44:40 UTC
spark git commit: [SPARK-20946][SQL] Do not update conf for existing
SparkContext in SparkSession.getOrCreate
Repository: spark
Updated Branches:
refs/heads/master 0217dfd26 -> 40c7add3a
[SPARK-20946][SQL] Do not update conf for existing SparkContext in SparkSession.getOrCreate
## What changes were proposed in this pull request?
SparkContext is shared by all sessions, we should not update its conf for only one session.
## How was this patch tested?
existing tests
Author: Wenchen Fan <we...@databricks.com>
Closes #18536 from cloud-fan/config.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/40c7add3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/40c7add3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/40c7add3
Branch: refs/heads/master
Commit: 40c7add3a4c811202d1fa2be9606aca08df81266
Parents: 0217dfd
Author: Wenchen Fan <we...@databricks.com>
Authored: Fri Jul 7 08:44:31 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Jul 7 08:44:31 2017 +0800
----------------------------------------------------------------------
.../spark/ml/recommendation/ALSSuite.scala | 4 +---
.../apache/spark/ml/tree/impl/TreeTests.scala | 2 --
.../org/apache/spark/sql/SparkSession.scala | 19 +++++++------------
.../spark/sql/SparkSessionBuilderSuite.scala | 8 +++-----
4 files changed, 11 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/40c7add3/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 3094f52..b57fc8d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -818,15 +818,13 @@ class ALSCleanerSuite extends SparkFunSuite {
FileUtils.listFiles(localDir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE).asScala.toSet
try {
conf.set("spark.local.dir", localDir.getAbsolutePath)
- val sc = new SparkContext("local[2]", "test", conf)
+ val sc = new SparkContext("local[2]", "ALSCleanerSuite", conf)
try {
sc.setCheckpointDir(checkpointDir.getAbsolutePath)
// Generate test data
val (training, _) = ALSSuite.genImplicitTestData(sc, 20, 5, 1, 0.2, 0)
// Implicitly test the cleaning of parents during ALS training
val spark = SparkSession.builder
- .master("local[2]")
- .appName("ALSCleanerSuite")
.sparkContext(sc)
.getOrCreate()
import spark.implicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/40c7add3/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
index 92a2369..b6894b3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
@@ -43,8 +43,6 @@ private[ml] object TreeTests extends SparkFunSuite {
categoricalFeatures: Map[Int, Int],
numClasses: Int): DataFrame = {
val spark = SparkSession.builder()
- .master("local[2]")
- .appName("TreeTests")
.sparkContext(data.sparkContext)
.getOrCreate()
import spark.implicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/40c7add3/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index 0ddcd21..6dfe8a6 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -867,7 +867,7 @@ object SparkSession {
*
* @since 2.2.0
*/
- def withExtensions(f: SparkSessionExtensions => Unit): Builder = {
+ def withExtensions(f: SparkSessionExtensions => Unit): Builder = synchronized {
f(extensions)
this
}
@@ -912,21 +912,16 @@ object SparkSession {
// No active nor global default session. Create a new one.
val sparkContext = userSuppliedContext.getOrElse {
- // set app name if not given
- val randomAppName = java.util.UUID.randomUUID().toString
val sparkConf = new SparkConf()
options.foreach { case (k, v) => sparkConf.set(k, v) }
+
+ // set a random app name if not given.
if (!sparkConf.contains("spark.app.name")) {
- sparkConf.setAppName(randomAppName)
- }
- val sc = SparkContext.getOrCreate(sparkConf)
- // maybe this is an existing SparkContext, update its SparkConf which maybe used
- // by SparkSession
- options.foreach { case (k, v) => sc.conf.set(k, v) }
- if (!sc.conf.contains("spark.app.name")) {
- sc.conf.setAppName(randomAppName)
+ sparkConf.setAppName(java.util.UUID.randomUUID().toString)
}
- sc
+
+ SparkContext.getOrCreate(sparkConf)
+ // Do not update `SparkConf` for existing `SparkContext`, as it's shared by all sessions.
}
// Initialize extensions if the user has defined a configurator class.
http://git-wip-us.apache.org/repos/asf/spark/blob/40c7add3/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
index cdac682..770e156 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SparkSessionBuilderSuite.scala
@@ -102,11 +102,9 @@ class SparkSessionBuilderSuite extends SparkFunSuite {
assert(session.conf.get("key1") == "value1")
assert(session.conf.get("key2") == "value2")
assert(session.sparkContext == sparkContext2)
- assert(session.sparkContext.conf.get("key1") == "value1")
- // If the created sparkContext is not passed through the Builder's API sparkContext,
- // the conf of this sparkContext will also contain the conf set through the API config.
- assert(session.sparkContext.conf.get("key2") == "value2")
- assert(session.sparkContext.conf.get("spark.app.name") == "test")
+ // We won't update conf for existing `SparkContext`
+ assert(!sparkContext2.conf.contains("key2"))
+ assert(sparkContext2.conf.get("key1") == "value1")
session.stop()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org