You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lr...@apache.org on 2018/04/21 20:36:31 UTC
[1/3] incubator-toree git commit: [TOREE-471] Fix typo in command
line configuration option
Repository: incubator-toree
Updated Branches:
refs/heads/master 284235c2b -> c88264aa4
[TOREE-471] Fix typo in command line configuration option
Fix typo in spark_context_intialization_timeout config option
spark_context_intialization_timeout => spark_context_initialization_timeout
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/5f4d77aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/5f4d77aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/5f4d77aa
Branch: refs/heads/master
Commit: 5f4d77aa434db464ee630befe4342c65af6b0149
Parents: 284235c
Author: Luciano Resende <lr...@apache.org>
Authored: Thu Apr 19 12:23:58 2018 -0700
Committer: Luciano Resende <lr...@apache.org>
Committed: Thu Apr 19 20:15:51 2018 -0700
----------------------------------------------------------------------
.../apache/toree/boot/CommandLineOptions.scala | 18 +++++++++---------
.../org/apache/toree/kernel/api/Kernel.scala | 4 ++--
.../toree/boot/CommandLineOptionsSpec.scala | 14 +++++++-------
.../org/apache/toree/kernel/api/KernelSpec.scala | 6 +++---
resources/compile/application.conf | 2 +-
resources/compile/reference.conf | 4 ++--
resources/test/application.conf | 2 +-
resources/test/reference.conf | 4 ++--
8 files changed, 27 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 13325a8..87e9578 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -70,11 +70,11 @@ class CommandLineOptions(args: Seq[String]) {
.withRequiredArg().ofType(classOf[String])
private val _default_repositories = parser.accepts(
- "default-repositories", "comma seperated list of additional repositories to resolve"
+ "default-repositories", "comma separated list of additional repositories to resolve"
).withRequiredArg().ofType(classOf[String])
private val _default_repository_credentials = parser.accepts(
- "default-repository-credentials", "comma seperated list of credential files to use"
+ "default-repository-credentials", "comma separated list of credential files to use"
).withRequiredArg().ofType(classOf[String])
private val _max_interpreter_threads = parser.accepts(
@@ -94,17 +94,17 @@ class CommandLineOptions(args: Seq[String]) {
private val _nosparkcontext =
parser.accepts("nosparkcontext", "kernel should not create a spark context")
- private val _interpreter_plugin = parser.accepts(
- "interpreter-plugin"
- ).withRequiredArg().ofType(classOf[String])
-
- private val _spark_context_intialization_timeout = parser.accepts(
- "spark-context-intialization-timeout",
+ private val _spark_context_initialization_timeout = parser.accepts(
+ "spark-context-initialization-timeout",
"The time (in milliseconds) allowed for creation of the spark context. " +
"Failure to create a context in this time could result in duplicate initialization messages. " +
"The default value is 100 milliseconds."
).withRequiredArg().ofType(classOf[Long])
+ private val _interpreter_plugin = parser.accepts(
+ "interpreter-plugin"
+ ).withRequiredArg().ofType(classOf[String])
+
private val _alternate_sigint = parser.accepts(
"alternate-sigint",
"Specifies the signal to use instead of SIGINT for interrupting a long-running cell. " +
@@ -160,11 +160,11 @@ class CommandLineOptions(args: Seq[String]) {
"magic_urls" -> getAll(_magic_url).map(_.asJava)
.flatMap(list => if (list.isEmpty) None else Some(list)),
"max_interpreter_threads" -> get(_max_interpreter_threads),
- "spark_context_intialization_timeout" -> get(_spark_context_intialization_timeout),
"alternate_sigint" -> get(_alternate_sigint),
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
"nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
+ "spark_context_initialization_timeout" -> get(_spark_context_initialization_timeout),
"interpreter_plugins" -> interpreterPlugins,
"default_repositories" -> getAll(_default_repositories).map(_.asJava)
.flatMap(list => if (list.isEmpty) None else Some(list)),
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
index 6900155..df6fed5 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -394,9 +394,9 @@ class Kernel (
// TODO: Exposed for testing purposes.
protected[toree] def getSparkContextInitializationTimeout: Long = {
- val timeout:Long = config.getDuration("spark_context_intialization_timeout", TimeUnit.MILLISECONDS)
+ val timeout:Long = config.getDuration("spark_context_initialization_timeout", TimeUnit.MILLISECONDS)
if (timeout <= 0) {
- val clOptionName = "spark-context-intialization-timeout"
+ val clOptionName = "spark_context_initialization_timeout"
throw new RuntimeException(s"--$clOptionName: Invalid timeout of '$timeout' milliseconds specified. " +
s"Must specify a positive value.")
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index 470b091..1363a92 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -187,7 +187,7 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
config.getInt("iopub_port") should be(43462)
config.getInt("control_port") should be(44808)
config.getInt("max_interpreter_threads") should be(4)
- config.getInt("spark_context_intialization_timeout") should be(100)
+ config.getInt("spark_context_initialization_timeout") should be(100)
}
}
}
@@ -283,8 +283,8 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
}
}
- describe("when dealing with --spark-context-intialization-timeout") {
- val key = "spark_context_intialization_timeout"
+ describe("when dealing with --spark-context-initialization-timeout") {
+ val key = "spark_context_initialization_timeout"
it("when none of the options are specified, it should default to 100") {
val options = new CommandLineOptions(Nil)
@@ -308,7 +308,7 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
"--iopub-port", "77777",
"--control-port", "55555",
"--heartbeat-port", "44444",
- "--spark-context-intialization-timeout", "30000"
+ "--spark-context-initialization-timeout", "30000"
))
val config: Config = options.toConfig
config.getInt(key) should be(30000)
@@ -321,7 +321,7 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
"--iopub-port", "77777",
"--control-port", "55555",
"--heartbeat-port", "44444",
- "--spark-context-intialization-timeout", "-1"
+ "--spark-context-initialization-timeout", "-1"
))
val config: Config = options.toConfig
config.getInt(key) should be(-1)
@@ -335,7 +335,7 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
"--iopub-port", "77777",
"--control-port", "55555",
"--heartbeat-port", "44444",
- "--spark-context-intialization-timeout", "foo"
+ "--spark-context-initialization-timeout", "foo"
))
val config: Config = options.toConfig
}
@@ -349,7 +349,7 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
"--iopub-port", "77777",
"--control-port", "55555",
"--heartbeat-port", "44444",
- "--spark-context-intialization-timeout", ""
+ "--spark-context-initialization-timeout", ""
))
val config: Config = options.toConfig
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
index de8d9d7..8bedfe3 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
@@ -209,7 +209,7 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
it("should use the specified timeout to initialize spark context") {
val expectedTimeout: Long = 30000
- doReturn(expectedTimeout).when(mockConfig).getDuration("spark_context_intialization_timeout", TimeUnit.MILLISECONDS)
+ doReturn(expectedTimeout).when(mockConfig).getDuration("spark_context_initialization_timeout", TimeUnit.MILLISECONDS)
kernel.getSparkContextInitializationTimeout should be(expectedTimeout)
}
@@ -217,7 +217,7 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
it("should throw an exception when negative value is specified as timeout") {
intercept[RuntimeException] {
val timeout: Long = -30000
- doReturn(timeout).when(mockConfig).getDuration("spark_context_intialization_timeout", TimeUnit.MILLISECONDS)
+ doReturn(timeout).when(mockConfig).getDuration("spark_context_initialization_timeout", TimeUnit.MILLISECONDS)
kernel.getSparkContextInitializationTimeout
}
@@ -226,7 +226,7 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
it("should throw an exception when zero is specified as timeout") {
intercept[RuntimeException] {
val timeout: Long = 0
- doReturn(timeout).when(mockConfig).getDuration("spark_context_intialization_timeout", TimeUnit.MILLISECONDS)
+ doReturn(timeout).when(mockConfig).getDuration("spark_context_initialization_timeout", TimeUnit.MILLISECONDS)
kernel.getSparkContextInitializationTimeout
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/resources/compile/application.conf
----------------------------------------------------------------------
diff --git a/resources/compile/application.conf b/resources/compile/application.conf
index 8c38f7d..1e0f5cc 100644
--- a/resources/compile/application.conf
+++ b/resources/compile/application.conf
@@ -26,4 +26,4 @@ control_port = ${?PORT1}
hb_port = ${?PORT2}
shell_port = ${?PORT3}
iopub_port = ${?PORT4}
-spark_context_intialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
+spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index 34bcd6d..b99f703 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -55,8 +55,8 @@ deps_dir = ${?DEPS_DIR}
default_interpreter = "Scala"
default_interpreter = ${?DEFAULT_INTERPRETER}
-spark_context_intialization_timeout = 100
-spark_context_intialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
+spark_context_initialization_timeout = 100
+spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
default_interpreter_plugin = [
"Scala:org.apache.toree.kernel.interpreter.scala.ScalaInterpreter",
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/resources/test/application.conf
----------------------------------------------------------------------
diff --git a/resources/test/application.conf b/resources/test/application.conf
index 8c38f7d..1e0f5cc 100644
--- a/resources/test/application.conf
+++ b/resources/test/application.conf
@@ -26,4 +26,4 @@ control_port = ${?PORT1}
hb_port = ${?PORT2}
shell_port = ${?PORT3}
iopub_port = ${?PORT4}
-spark_context_intialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
+spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/5f4d77aa/resources/test/reference.conf
----------------------------------------------------------------------
diff --git a/resources/test/reference.conf b/resources/test/reference.conf
index 5f06560..3324442 100644
--- a/resources/test/reference.conf
+++ b/resources/test/reference.conf
@@ -54,8 +54,8 @@ send_empty_output = ${?SEND_EMPTY_OUTPUT}
default_interpreter = "Scala"
default_interpreter = ${?DEFAULT_INTERPRETER}
-spark_context_intialization_timeout = 100
-spark_context_intialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
+spark_context_initialization_timeout = 100
+spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
default_interpreter_plugin = [
"Scala:org.apache.toree.kernel.interpreter.scala.ScalaInterpreter",
[3/3] incubator-toree git commit: [MINOR] Ignore test with external
dependencies
Posted by lr...@apache.org.
[MINOR] Ignore test with external dependencies
There are intermitent failure when downloading external
java based jar for testing %addJar functionality.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/c88264aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/c88264aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/c88264aa
Branch: refs/heads/master
Commit: c88264aa4ab973ad0d5cd708ee6f849eeda2bb41
Parents: 9c929f5
Author: Luciano Resende <lr...@apache.org>
Authored: Sat Apr 21 13:35:06 2018 -0700
Committer: Luciano Resende <lr...@apache.org>
Committed: Sat Apr 21 13:35:06 2018 -0700
----------------------------------------------------------------------
.../interpreter/scala/AddExternalJarMagicSpecForIntegration.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/c88264aa/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
----------------------------------------------------------------------
diff --git a/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala b/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
index b6834e1..5fe29ff 100644
--- a/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
+++ b/scala-interpreter/src/test/scala/integration/interpreter/scala/AddExternalJarMagicSpecForIntegration.scala
@@ -91,7 +91,7 @@ class AddExternalJarMagicSpecForIntegration
outputResult.reset()
}
- it("should support Scala jars") {
+ ignore("should support Scala jars") {
val locationURL = "http://repo1.maven.org/maven2/org/scala-rules/rule-engine-core_2.11/0.5.1/rule-engine-core_2.11-0.5.1.jar"
val testJarUrl = JarUtils.downloadJar(tempdir.toString, locationURL)
[2/3] incubator-toree git commit: [TOREE-470] Config option do
control SparkContext initialization
Posted by lr...@apache.org.
[TOREE-470] Config option do control SparkContext initialization
--spark-context-initialization-mode eager (disable lazy initialization)
--spark-context-initialization-mode lazy (default, enable lazy initialization)
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/9c929f51
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/9c929f51
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/9c929f51
Branch: refs/heads/master
Commit: 9c929f517ba0617ada3a782aa9c28826f82a8e9f
Parents: 5f4d77a
Author: Luciano Resende <lr...@apache.org>
Authored: Thu Apr 19 20:12:59 2018 -0700
Committer: Luciano Resende <lr...@apache.org>
Committed: Fri Apr 20 12:19:31 2018 -0700
----------------------------------------------------------------------
.../apache/toree/boot/CommandLineOptions.scala | 10 ++++
.../boot/layer/ComponentInitialization.scala | 8 +++
.../org/apache/toree/kernel/api/Kernel.scala | 57 +++++++++++--------
.../toree/boot/CommandLineOptionsSpec.scala | 60 ++++++++++++++++++++
resources/compile/application.conf | 1 +
resources/compile/reference.conf | 5 +-
resources/test/application.conf | 1 +
resources/test/reference.conf | 3 +
8 files changed, 119 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 87e9578..ae7f220 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -21,6 +21,7 @@ import java.io.{File, OutputStream}
import com.typesafe.config.{Config, ConfigFactory}
import joptsimple.{OptionParser, OptionSpec}
+import joptsimple.util.RegexMatcher._
import scala.collection.JavaConverters._
@@ -94,6 +95,13 @@ class CommandLineOptions(args: Seq[String]) {
private val _nosparkcontext =
parser.accepts("nosparkcontext", "kernel should not create a spark context")
+ private val _spark_context_initialization_mode = parser.accepts(
+ "spark-context-initialization-mode",
+ "Identify how the Spark context initialization occurs. " +
+ "EAGER initialization will happen during runtime initialization, " +
+ "LAZY initialization will happen when the context is used for the first time ."
+ ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(lazy)|(eager)")).defaultsTo("lazy")
+
private val _spark_context_initialization_timeout = parser.accepts(
"spark-context-initialization-timeout",
"The time (in milliseconds) allowed for creation of the spark context. " +
@@ -164,6 +172,8 @@ class CommandLineOptions(args: Seq[String]) {
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
"nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
+ "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode))
+ get(_spark_context_initialization_mode) else Some("lazy")),
"spark_context_initialization_timeout" -> get(_spark_context_initialization_timeout),
"interpreter_plugins" -> interpreterPlugins,
"default_repositories" -> getAll(_default_repositories).map(_.asJava)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
index 42999d7..46f0796 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/layer/ComponentInitialization.scala
@@ -82,6 +82,8 @@ trait StandardComponentInitialization extends ComponentInitialization {
initializePlugins(config, pluginManager)
+ initializeSparkContext(config, kernel)
+
interpreterManager.initializeInterpreters(kernel)
pluginManager.fireEvent(AllInterpretersReady)
@@ -108,6 +110,12 @@ trait StandardComponentInitialization extends ComponentInitialization {
(commStorage, commRegistrar, commManager)
}
+ def initializeSparkContext(config:Config, kernel:Kernel) = {
+ if(config.getString("spark_context_initialization_mode") == "eager") {
+ kernel.sparkSession
+ }
+ }
+
private def initializeDependencyDownloader(config: Config) = {
val depsDir = {
if(config.hasPath("deps_dir") && Files.exists(Paths.get(config.getString("deps_dir")))) {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
index df6fed5..cb90b81 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -410,32 +410,39 @@ class Kernel (
private lazy val defaultSparkConf: SparkConf = createSparkConf(new SparkConf())
override def sparkSession: SparkSession = {
- defaultSparkConf.getOption("spark.master") match {
- case Some(master) if !master.contains("local") =>
- // When connecting to a remote cluster, the first call to getOrCreate
- // may create a session and take a long time, so this starts a future
- // to get the session. If it take longer than specified timeout, then
- // print a message to the user that Spark is starting. Note, the
- // default timeout is 100ms and it is specified in reference.conf.
- import scala.concurrent.ExecutionContext.Implicits.global
- val sessionFuture = Future {
+
+ if(config.getString("spark_context_initialization_mode") == "eager") {
+ // explicitly enable eager initialization of spark context
+ SparkSession.builder.config(defaultSparkConf).getOrCreate
+ } else {
+ // default lazy initialization of spark context
+ defaultSparkConf.getOption("spark.master") match {
+ case Some(master) if !master.contains("local") =>
+ // When connecting to a remote cluster, the first call to getOrCreate
+ // may create a session and take a long time, so this starts a future
+ // to get the session. If it take longer than specified timeout, then
+ // print a message to the user that Spark is starting. Note, the
+ // default timeout is 100ms and it is specified in reference.conf.
+ import scala.concurrent.ExecutionContext.Implicits.global
+ val sessionFuture = Future {
+ SparkSession.builder.config(defaultSparkConf).getOrCreate
+ }
+
+ try {
+ val timeout = getSparkContextInitializationTimeout
+ Await.result(sessionFuture, Duration(timeout, TimeUnit.MILLISECONDS))
+ } catch {
+ case timeout: TimeoutException =>
+ // getting the session is taking a long time, so assume that Spark
+ // is starting and print a message
+ display.content(
+ MIMEType.PlainText, "Waiting for a Spark session to start...")
+ Await.result(sessionFuture, Duration.Inf)
+ }
+
+ case _ =>
SparkSession.builder.config(defaultSparkConf).getOrCreate
- }
-
- try {
- val timeout = getSparkContextInitializationTimeout
- Await.result(sessionFuture, Duration(timeout, TimeUnit.MILLISECONDS))
- } catch {
- case timeout: TimeoutException =>
- // getting the session is taking a long time, so assume that Spark
- // is starting and print a message
- display.content(
- MIMEType.PlainText, "Waiting for a Spark session to start...")
- Await.result(sessionFuture, Duration.Inf)
- }
-
- case _ =>
- SparkSession.builder.config(defaultSparkConf).getOrCreate
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index 1363a92..029f46b 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -356,6 +356,66 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
}
}
+ describe("when dealing with --spark-context-initialization-mode") {
+ val key = "spark_context_initialization_mode"
+
+ it("when none of the options are specified, it should default to lazy") {
+ val options = new CommandLineOptions(Nil)
+ val config: Config = options.toConfig
+ config.getString(key) should be("lazy")
+ }
+
+ it("when other options are specified, it should default to lazy") {
+ val options = new CommandLineOptions(Seq(
+ "--interpreter-plugin",
+ "dummy:test.utils.DummyInterpreter"
+ ))
+ val config: Config = options.toConfig
+ config.getString(key) should be("lazy")
+ }
+
+ it("when the options is specified, it should return the specified value") {
+ val options = new CommandLineOptions(List(
+ "--stdin-port", "99999",
+ "--shell-port", "88888",
+ "--iopub-port", "77777",
+ "--control-port", "55555",
+ "--heartbeat-port", "44444",
+ "--spark-context-initialization-mode", "eager"
+ ))
+ val config: Config = options.toConfig
+ config.getString(key) should be("eager")
+ }
+
+ it("when an invalid value is specified, an exception must be thrown") {
+ intercept [OptionException] {
+ val options = new CommandLineOptions(List(
+ "--stdin-port", "99999",
+ "--shell-port", "88888",
+ "--iopub-port", "77777",
+ "--control-port", "55555",
+ "--heartbeat-port", "44444",
+ "--spark-context-initialization-mode", "foo"
+ ))
+ val config: Config = options.toConfig
+ }
+ }
+
+ it("when a value is not specified, an exception must be thrown") {
+ intercept [OptionException] {
+ val options = new CommandLineOptions(List(
+ "--stdin-port", "99999",
+ "--shell-port", "88888",
+ "--iopub-port", "77777",
+ "--control-port", "55555",
+ "--heartbeat-port", "44444",
+ "--spark-context-initialization-mode", ""
+ ))
+ val config: Config = options.toConfig
+ }
+ }
+ }
+
describe("when dealing with --alternate-sigint") {
val key = "alternate_sigint"
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/resources/compile/application.conf
----------------------------------------------------------------------
diff --git a/resources/compile/application.conf b/resources/compile/application.conf
index 1e0f5cc..3f74bd9 100644
--- a/resources/compile/application.conf
+++ b/resources/compile/application.conf
@@ -26,4 +26,5 @@ control_port = ${?PORT1}
hb_port = ${?PORT2}
shell_port = ${?PORT3}
iopub_port = ${?PORT4}
+spark_context_initialization_mode = ${?SPARK_CONTEXT_INITIALIZATION_MODE}
spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index b99f703..1c98865 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -55,8 +55,11 @@ deps_dir = ${?DEPS_DIR}
default_interpreter = "Scala"
default_interpreter = ${?DEFAULT_INTERPRETER}
+spark_context_initialization_mode = "lazy"
+spark_context_initialization_mode = ${?SPARK_CONTEXT_INITIALIZATION_MODE}
+
spark_context_initialization_timeout = 100
-spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
+spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_MODE}
default_interpreter_plugin = [
"Scala:org.apache.toree.kernel.interpreter.scala.ScalaInterpreter",
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/resources/test/application.conf
----------------------------------------------------------------------
diff --git a/resources/test/application.conf b/resources/test/application.conf
index 1e0f5cc..3f74bd9 100644
--- a/resources/test/application.conf
+++ b/resources/test/application.conf
@@ -26,4 +26,5 @@ control_port = ${?PORT1}
hb_port = ${?PORT2}
shell_port = ${?PORT3}
iopub_port = ${?PORT4}
+spark_context_initialization_mode = ${?SPARK_CONTEXT_INITIALIZATION_MODE}
spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/9c929f51/resources/test/reference.conf
----------------------------------------------------------------------
diff --git a/resources/test/reference.conf b/resources/test/reference.conf
index 3324442..66741d4 100644
--- a/resources/test/reference.conf
+++ b/resources/test/reference.conf
@@ -54,6 +54,9 @@ send_empty_output = ${?SEND_EMPTY_OUTPUT}
default_interpreter = "Scala"
default_interpreter = ${?DEFAULT_INTERPRETER}
+spark_context_initialization_mode = "lazy"
+spark_context_initialization_mode = ${?SPARK_CONTEXT_INITIALIZATION_MODE}
+
spark_context_initialization_timeout = 100
spark_context_initialization_timeout = ${?SPARK_CONTEXT_INITIALIZATION_TIMEOUT}