You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/06/02 17:05:08 UTC
spark git commit: [SPARK-20946][SQL] simplify the config setting
logic in SparkSession.getOrCreate
Repository: spark
Updated Branches:
refs/heads/master d1b80ab92 -> e11d90bf8
[SPARK-20946][SQL] simplify the config setting logic in SparkSession.getOrCreate
## What changes were proposed in this pull request?
The current conf setting logic is a little complex and has duplication, this PR simplifies it.
## How was this patch tested?
existing tests.
Author: Wenchen Fan <we...@databricks.com>
Closes #18172 from cloud-fan/session.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e11d90bf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e11d90bf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e11d90bf
Branch: refs/heads/master
Commit: e11d90bf8deb553fd41b8837e3856c11486c2503
Parents: d1b80ab
Author: Wenchen Fan <we...@databricks.com>
Authored: Fri Jun 2 10:05:05 2017 -0700
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Jun 2 10:05:05 2017 -0700
----------------------------------------------------------------------
.../spark/ml/recommendation/ALSSuite.scala | 4 +---
.../apache/spark/ml/tree/impl/TreeTests.scala | 2 --
.../org/apache/spark/sql/SparkSession.scala | 25 +++++++-------------
3 files changed, 10 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e11d90bf/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index 701040f..23f2256 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -820,15 +820,13 @@ class ALSCleanerSuite extends SparkFunSuite {
FileUtils.listFiles(localDir, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE).asScala.toSet
try {
conf.set("spark.local.dir", localDir.getAbsolutePath)
- val sc = new SparkContext("local[2]", "test", conf)
+ val sc = new SparkContext("local[2]", "ALSCleanerSuite", conf)
try {
sc.setCheckpointDir(checkpointDir.getAbsolutePath)
// Generate test data
val (training, _) = ALSSuite.genImplicitTestData(sc, 20, 5, 1, 0.2, 0)
// Implicitly test the cleaning of parents during ALS training
val spark = SparkSession.builder
- .master("local[2]")
- .appName("ALSCleanerSuite")
.sparkContext(sc)
.getOrCreate()
import spark.implicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/e11d90bf/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
index 92a2369..b6894b3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/TreeTests.scala
@@ -43,8 +43,6 @@ private[ml] object TreeTests extends SparkFunSuite {
categoricalFeatures: Map[Int, Int],
numClasses: Int): DataFrame = {
val spark = SparkSession.builder()
- .master("local[2]")
- .appName("TreeTests")
.sparkContext(data.sparkContext)
.getOrCreate()
import spark.implicits._
http://git-wip-us.apache.org/repos/asf/spark/blob/e11d90bf/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
index d2bf350..bf37b76 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala
@@ -757,6 +757,8 @@ object SparkSession {
private[this] var userSuppliedContext: Option[SparkContext] = None
+ // The `SparkConf` inside the given `SparkContext` may get changed if you specify some options
+ // for this builder.
private[spark] def sparkContext(sparkContext: SparkContext): Builder = synchronized {
userSuppliedContext = Option(sparkContext)
this
@@ -854,7 +856,7 @@ object SparkSession {
*
* @since 2.2.0
*/
- def withExtensions(f: SparkSessionExtensions => Unit): Builder = {
+ def withExtensions(f: SparkSessionExtensions => Unit): Builder = synchronized {
f(extensions)
this
}
@@ -899,22 +901,14 @@ object SparkSession {
// No active nor global default session. Create a new one.
val sparkContext = userSuppliedContext.getOrElse {
- // set app name if not given
- val randomAppName = java.util.UUID.randomUUID().toString
val sparkConf = new SparkConf()
- options.foreach { case (k, v) => sparkConf.set(k, v) }
- if (!sparkConf.contains("spark.app.name")) {
- sparkConf.setAppName(randomAppName)
- }
- val sc = SparkContext.getOrCreate(sparkConf)
- // maybe this is an existing SparkContext, update its SparkConf which maybe used
- // by SparkSession
- options.foreach { case (k, v) => sc.conf.set(k, v) }
- if (!sc.conf.contains("spark.app.name")) {
- sc.conf.setAppName(randomAppName)
- }
- sc
+ options.get("spark.master").foreach(sparkConf.setMaster)
+ // set a random app name if not given.
+ sparkConf.setAppName(options.getOrElse("spark.app.name",
+ java.util.UUID.randomUUID().toString))
+ SparkContext.getOrCreate(sparkConf)
}
+ options.foreach { case (k, v) => sparkContext.conf.set(k, v) }
// Initialize extensions if the user has defined a configurator class.
val extensionConfOption = sparkContext.conf.get(StaticSQLConf.SPARK_SESSION_EXTENSIONS)
@@ -935,7 +929,6 @@ object SparkSession {
}
session = new SparkSession(sparkContext, None, None, extensions)
- options.foreach { case (k, v) => session.sessionState.conf.setConfString(k, v) }
defaultSession.set(session)
// Register a successfully instantiated context to the singleton. This should be at the
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org