You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by lb...@apache.org on 2016/03/30 18:39:41 UTC
incubator-toree git commit: Removing --spark-configuration cmd arg
Repository: incubator-toree
Updated Branches:
refs/heads/master 2aa8ce12b -> 7129ce222
Removing --spark-configuration cmd arg
Project: http://git-wip-us.apache.org/repos/asf/incubator-toree/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-toree/commit/7129ce22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-toree/tree/7129ce22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-toree/diff/7129ce22
Branch: refs/heads/master
Commit: 7129ce222521141b747f0a490e8228be5f417f93
Parents: 2aa8ce1
Author: Gino Bustelo <lb...@apache.org>
Authored: Wed Mar 30 11:19:12 2016 -0500
Committer: Gino Bustelo <lb...@apache.org>
Committed: Wed Mar 30 11:19:12 2016 -0500
----------------------------------------------------------------------
Vagrantfile | 3 +-
.../apache/toree/boot/CommandLineOptions.scala | 8 ----
.../org/apache/toree/kernel/api/Kernel.scala | 7 ----
.../toree/boot/CommandLineOptionsSpec.scala | 44 --------------------
.../apache/toree/kernel/api/KernelSpec.scala | 1 -
resources/compile/reference.conf | 3 --
resources/test/reference.conf | 3 --
7 files changed, 1 insertion(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/Vagrantfile
----------------------------------------------------------------------
diff --git a/Vagrantfile b/Vagrantfile
index 499194f..2f9d804 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -130,9 +130,8 @@ cat << EOF > /home/vagrant/.ipython/kernels/toree-kernel/kernel.json
],
"codemirror_mode": "scala",
"env": {
- "SPARK_OPTS": "--driver-java-options=-Xms1024M --driver-java-options=-Xmx4096M --driver-java-options=-Dlog4j.logLevel=trace",
+ "SPARK_OPTS": "--conf=spark.cores.max=4 --driver-java-options='-Xms1024M -Xmx4096M -Dlog4j.logLevel=trace'",
"MAX_INTERPRETER_THREADS": "16",
- "SPARK_CONFIGURATION": "spark.cores.max=4",
"CAPTURE_STANDARD_OUT": "true",
"CAPTURE_STANDARD_ERR": "true",
"SEND_EMPTY_OUTPUT": "false",
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
index 68d2458..da3ef49 100644
--- a/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
+++ b/kernel/src/main/scala/org/apache/toree/boot/CommandLineOptions.scala
@@ -67,11 +67,6 @@ class CommandLineOptions(args: Seq[String]) {
"heartbeat-port", "port of the heartbeat socket"
).withRequiredArg().ofType(classOf[Int])
- private val _spark_configuration = parser.acceptsAll(
- Seq("spark-configuration", "S").asJava,
- "configuration setting for Apache Spark"
- ).withRequiredArg().ofType(classOf[KeyValuePair])
-
private val _magic_url =
parser.accepts("magic-url", "path to a magic jar")
.withRequiredArg().ofType(classOf[String])
@@ -145,9 +140,6 @@ class CommandLineOptions(args: Seq[String]) {
"interpreter_args" -> interpreterArgs,
"magic_urls" -> getAll(_magic_url).map(_.asJava)
.flatMap(list => if (list.isEmpty) None else Some(list)),
- "spark_configuration" -> getAll(_spark_configuration)
- .map(list => KeyValuePairUtils.keyValuePairSeqToString(list))
- .flatMap(str => if (str.nonEmpty) Some(str) else None),
"max_interpreter_threads" -> get(_max_interpreter_threads),
"jar_dir" -> get(_jar_dir),
"default_interpreter" -> get(_default_interpreter),
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
----------------------------------------------------------------------
diff --git a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
index 2d67189..bf7f56a 100644
--- a/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
+++ b/kernel/src/main/scala/org/apache/toree/kernel/api/Kernel.scala
@@ -382,13 +382,6 @@ class Kernel (
logger.info("Setting deployMode to client")
conf.set("spark.submit.deployMode", "client")
- KeyValuePairUtils.stringToKeyValuePairSeq(
- _config.getString("spark_configuration")
- ).foreach { keyValuePair =>
- logger.info(s"Setting ${keyValuePair.key} to ${keyValuePair.value}")
- Try(conf.set(keyValuePair.key, keyValuePair.value))
- }
-
// TODO: Move SparkIMain to private and insert in a different way
logger.warn("Locked to Scala interpreter with SparkIMain until decoupled!")
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
index cda360a..f88dd7d 100644
--- a/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/boot/CommandLineOptionsSpec.scala
@@ -89,50 +89,6 @@ class CommandLineOptionsSpec extends FunSpec with Matchers {
}
}
- describe("when received --spark-conf=<key>=<value>") {
- it("should add the key-value pair to the string representation") {
- val expected = "key=value"
- val options = new CommandLineOptions(s"--spark-conf=$expected" :: Nil)
- val actual = options.toConfig.getString("spark_configuration")
-
- actual should be (expected)
- }
-
- it("should append to existing command line key-value pairs") {
- val expected = "key1=value1" :: "key2=value2" :: Nil
- val options = new CommandLineOptions(
- s"--spark-conf=${expected(0)}" ::
- s"--spark-conf=${expected(1)}" ::
- Nil
- )
- val actual = options.toConfig.getString("spark_configuration")
-
- actual should be (expected.mkString(","))
- }
- }
-
- describe("when received -S<key>=<value>") {
- it("should add the key-value pair to the string representation") {
- val expected = "key=value"
- val options = new CommandLineOptions(s"-S$expected" :: Nil)
- val actual = options.toConfig.getString("spark_configuration")
-
- actual should be(expected)
- }
-
- it("should append to existing command line key-value pairs") {
- val expected = "key1=value1" :: "key2=value2" :: Nil
- val options = new CommandLineOptions(
- s"-S${expected(0)}" ::
- s"-S${expected(1)}" ::
- Nil
- )
- val actual = options.toConfig.getString("spark_configuration")
-
- actual should be (expected.mkString(","))
- }
- }
-
describe("when received --profile=<path>") {
it("should error if path is not set") {
intercept[OptionException] {
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
----------------------------------------------------------------------
diff --git a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
index f8ae1eb..5f43c07 100644
--- a/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
+++ b/kernel/src/test/scala/org/apache/toree/kernel/api/KernelSpec.scala
@@ -197,7 +197,6 @@ class KernelSpec extends FunSpec with Matchers with MockitoSugar
it("should create SparkConf") {
val expected = "some value"
doReturn(expected).when(mockConfig).getString("spark.master")
- doReturn("").when(mockConfig).getString("spark_configuration")
// Provide stub for interpreter classServerURI since also executed
doReturn("").when(mockInterpreter).classServerURI
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/resources/compile/reference.conf
----------------------------------------------------------------------
diff --git a/resources/compile/reference.conf b/resources/compile/reference.conf
index 5857133..18efe52 100644
--- a/resources/compile/reference.conf
+++ b/resources/compile/reference.conf
@@ -45,9 +45,6 @@ interpreter_args = []
magic_urls = []
magic_urls = [${?MAGIC_URLS}]
-spark_configuration = ""
-spark_configuration = ${?SPARK_CONFIGURATION}
-
max_interpreter_threads = 4
max_interpreter_threads = ${?MAX_INTERPRETER_THREADS}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/7129ce22/resources/test/reference.conf
----------------------------------------------------------------------
diff --git a/resources/test/reference.conf b/resources/test/reference.conf
index 806a977..461d4e8 100644
--- a/resources/test/reference.conf
+++ b/resources/test/reference.conf
@@ -46,9 +46,6 @@ interpreter_args = []
magic_urls = []
magic_urls = [${?MAGIC_URLS}]
-spark_configuration = ""
-spark_configuration = ${?SPARK_CONFIGURATION}
-
max_interpreter_threads = 4
max_interpreter_threads = ${?MAX_INTERPRETER_THREADS}