You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lr...@apache.org on 2018/08/16 03:25:08 UTC
[1/2] incubator-toree git commit: [MINOR] Fix typo on release notes
Repository: incubator-toree
Updated Branches:
refs/heads/master b6a922ac5 -> b8cca2cb5
[MINOR] Fix typo on release notes
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/bff1c6ba
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/bff1c6ba
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/bff1c6ba
Branch: refs/heads/master
Commit: bff1c6bafbabefd26a40fa37e3a5cd1f48c66c59
Parents: b6a922a
Author: Luciano Resende <lr...@apache.org>
Authored: Wed Aug 15 20:23:54 2018 -0700
Committer: Luciano Resende <lr...@apache.org>
Committed: Wed Aug 15 20:23:54 2018 -0700
----------------------------------------------------------------------
RELEASE_NOTES.md | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/bff1c6ba/RELEASE_NOTES.md
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 960f64c..fffc7c4 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -23,12 +23,11 @@
* Support Apache Spark 2.x codebase including Spark 2.2.2
* Enable Toree to run in Yarn cluster mode
-* Support configurable alternative interrupt signal via --alternate-sigint command line
* Create spark context lazily to avoid long startup times for the kernel
* Properly cleanup of temporary files/directories upon kernel shutdown
* %AddJAR now supports HDFS file format
* %AddDEP now defaults to default configuration
-* Cell Interrupt now cancel running Spark jobs and works in background process
+* Cell Interrupt now cancel running Spark jobs and works in background processes
* Interpreters now have the ability to send results other than text/plain
[2/2] incubator-toree git commit: [TOREE-480] Add support for
spark-context-initialization-mode none
Posted by lr...@apache.org.
[TOREE-480] Add support for spark-context-initialization-mode none
Deprecate nosparkcontext in favor of spark-context-initialization-mode none.
Note that old usage of nosparkcontext is automatically converted to use
spark-context-initialization-mode none.
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/b8cca2cb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/b8cca2cb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/b8cca2cb
Branch: refs/heads/master
Commit: b8cca2cb55fc17359a6cb89477bb58bef0962196
Parents: bff1c6b
Author: Luciano Resende <lr...@apache.org>
Authored: Thu Aug 2 13:45:31 2018 -0700
Committer: Luciano Resende <lr...@apache.org>
Committed: Wed Aug 15 20:24:07 2018 -0700
----------------------------------------------------------------------
etc/pip_install/toree/toreeapp.py | 2 +-
.../org/apache/toree/boot/CommandLineOptions.scala | 14 ++++++++------
.../apache/toree/boot/CommandLineOptionsSpec.scala | 15 ++++++++++++++-
3 files changed, 23 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/etc/pip_install/toree/toreeapp.py
----------------------------------------------------------------------
diff --git a/etc/pip_install/toree/toreeapp.py b/etc/pip_install/toree/toreeapp.py
index 84dad4c..c4dc11d 100644
--- a/etc/pip_install/toree/toreeapp.py
+++ b/etc/pip_install/toree/toreeapp.py
@@ -49,7 +49,7 @@ class ToreeInstall(InstallKernelSpec):
jupyter toree install --spark_home=/spark/home/dir
jupyter toree install --spark_opts='--master=local[4]'
jupyter toree install --kernel_name=toree_special
- jupyter toree install --toree_opts='--nosparkcontext'
+ jupyter toree install --toree_opts='--spark-context-initialization-mode none'
jupyter toree install --interpreters=PySpark,SQL
jupyter toree install --python=python
'''
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index ae7f220..39376ae 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -93,14 +93,15 @@ class CommandLineOptions(args: Seq[String]) {
.withRequiredArg().ofType(classOf[String])
private val _nosparkcontext =
- parser.accepts("nosparkcontext", "kernel should not create a spark context")
+ parser.accepts("nosparkcontext", "Deprecated (see spark-context-initialization-mode). \n Kernel should not create a spark context")
private val _spark_context_initialization_mode = parser.accepts(
"spark-context-initialization-mode",
"Identify how the Spark context initialization occurs. " +
+ "NONE will not initialize a Spark Context," +
"EAGER initialization will happen during runtime initialization, " +
- "LAZY initialization will happen when the context is used for the first time ."
- ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(lazy)|(eager)")).defaultsTo("lazy")
+ "LAZY initialization will happen when the context is used for the first time."
+ ).withRequiredArg().ofType(classOf[String]).withValuesConvertedBy( regex("(none)|(lazy)|(eager)")).defaultsTo("lazy")
private val _spark_context_initialization_timeout = parser.accepts(
"spark-context-initialization-timeout",
@@ -171,9 +172,10 @@ class CommandLineOptions(args: Seq[String]) {
"alternate_sigint" -> get(_alternate_sigint),
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
- "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
- "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode))
- get(_spark_context_initialization_mode) else Some("lazy")),
+ // deprecated in favor of spark-context-initialization-mode none
+ // "nosparkcontext" -> (if (has(_nosparkcontext)) Some(true) else Some(false)),
+ "spark_context_initialization_mode" -> (if( has(_spark_context_initialization_mode)) get(_spark_context_initialization_mode)
+ else ( if (has(_nosparkcontext)) Some("none") else Some("lazy"))),
"spark_context_initialization_timeout" -> get(_spark_context_initialization_timeout),
"interpreter_plugins" -> interpreterPlugins,
"default_repositories" -> getAll(_default_repositories).map(_.asJava)
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/b8cca2cb/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index 029f46b..2ec7020 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -387,7 +387,20 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
config.getString(key) should be("eager")
}
- it("when an invalid value is specified, an exception must be thrown") {
+ it("when the option nosparkcontext is specified, it should properly set spark-context-initialization-mode to none") {
+ val options = new CommandLineOptions(List(
+ "--stdin-port", "99999",
+ "--shell-port", "88888",
+ "--iopub-port", "77777",
+ "--control-port", "55555",
+ "--heartbeat-port", "44444",
+ "--nosparkcontext", "foo"
+ ))
+ val config: Config = options.toConfig
+ config.getString(key) should be("none")
+ }
+
+ it("when an invalid value is specified, an exception must be thrown") {
intercept [OptionException] {
val options = new CommandLineOptions(List(
"--stdin-port", "99999",