You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2015/12/10 17:36:00 UTC
spark git commit: [SPARK-11832][CORE] Process arguments in
spark-shell for Scala 2.11
Repository: spark
Updated Branches:
refs/heads/master 76540b6df -> db5165246
[SPARK-11832][CORE] Process arguments in spark-shell for Scala 2.11
Process arguments passed to the spark-shell. Fixes running the spark-shell from within a build environment.
Author: Jakob Odersky <jo...@gmail.com>
Closes #9824 from jodersky/shell-2.11.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/db516524
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/db516524
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/db516524
Branch: refs/heads/master
Commit: db5165246f2888537dd0f3d4c5a515875c7358ed
Parents: 76540b6
Author: Jakob Odersky <jo...@gmail.com>
Authored: Thu Dec 10 08:35:52 2015 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Thu Dec 10 08:35:52 2015 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/repl/Main.scala | 37 ++++++++++++++------
.../scala/org/apache/spark/repl/ReplSuite.scala | 3 +-
2 files changed, 27 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/db516524/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
index 627148d..455a6b9 100644
--- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
+++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
@@ -31,24 +31,39 @@ object Main extends Logging {
val tmp = System.getProperty("java.io.tmpdir")
val rootDir = conf.get("spark.repl.classdir", tmp)
val outputDir = Utils.createTempDir(rootDir)
- val s = new Settings()
- s.processArguments(List("-Yrepl-class-based",
- "-Yrepl-outdir", s"${outputDir.getAbsolutePath}",
- "-classpath", getAddedJars.mkString(File.pathSeparator)), true)
// the creation of SecurityManager has to be lazy so SPARK_YARN_MODE is set if needed
lazy val classServer = new HttpServer(conf, outputDir, new SecurityManager(conf))
var sparkContext: SparkContext = _
var sqlContext: SQLContext = _
var interp = new SparkILoop // this is a public var because tests reset it.
+ private var hasErrors = false
+
+ private def scalaOptionError(msg: String): Unit = {
+ hasErrors = true
+ Console.err.println(msg)
+ }
+
def main(args: Array[String]) {
- if (getMaster == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true")
- // Start the classServer and store its URI in a spark system property
- // (which will be passed to executors so that they can connect to it)
- classServer.start()
- interp.process(s) // Repl starts and goes in loop of R.E.P.L
- classServer.stop()
- Option(sparkContext).map(_.stop)
+
+ val interpArguments = List(
+ "-Yrepl-class-based",
+ "-Yrepl-outdir", s"${outputDir.getAbsolutePath}",
+ "-classpath", getAddedJars.mkString(File.pathSeparator)
+ ) ++ args.toList
+
+ val settings = new Settings(scalaOptionError)
+ settings.processArguments(interpArguments, true)
+
+ if (!hasErrors) {
+ if (getMaster == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true")
+ // Start the classServer and store its URI in a spark system property
+ // (which will be passed to executors so that they can connect to it)
+ classServer.start()
+ interp.process(settings) // Repl starts and goes in loop of R.E.P.L
+ classServer.stop()
+ Option(sparkContext).map(_.stop)
+ }
}
def getAddedJars: Array[String] = {
http://git-wip-us.apache.org/repos/asf/spark/blob/db516524/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
----------------------------------------------------------------------
diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index bf89979..63f3688 100644
--- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -54,8 +54,7 @@ class ReplSuite extends SparkFunSuite {
new SparkILoop(in, new PrintWriter(out))
}
org.apache.spark.repl.Main.interp = interp
- Main.s.processArguments(List("-classpath", classpath), true)
- Main.main(Array()) // call main
+ Main.main(Array("-classpath", classpath)) // call main
org.apache.spark.repl.Main.interp = null
if (oldExecutorClasspath != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org