You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2014/06/10 08:07:30 UTC
git commit: Spark 1384 - Fix spark-shell on yarn access to secure
hdfs - branch-0.9 only
Repository: spark
Updated Branches:
refs/heads/branch-0.9 1d3aab961 -> cc95d978a
Spark 1384 - Fix spark-shell on yarn access to secure hdfs - branch-0.9 only
Author: Thomas Graves <tg...@apache.org>
Closes #287 from tgravescs/SPARK-1384 and squashes the following commits:
ae9162a [Thomas Graves] SPARK-1384 - fix spark-shell on yarn access to secure HDFS
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cc95d978
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cc95d978
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cc95d978
Branch: refs/heads/branch-0.9
Commit: cc95d978a1b2dab6ed3dd8f73ccf8c299b16fdc0
Parents: 1d3aab9
Author: Thomas Graves <tg...@apache.org>
Authored: Mon Jun 9 23:07:25 2014 -0700
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Mon Jun 9 23:07:25 2014 -0700
----------------------------------------------------------------------
.../org/apache/spark/repl/SparkILoop.scala | 21 ++++++++++++--------
1 file changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/cc95d978/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
index f262faa..5274932 100644
--- a/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
+++ b/repl/src/main/scala/org/apache/spark/repl/SparkILoop.scala
@@ -880,6 +880,7 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter,
def process(settings: Settings): Boolean = savingContextLoader {
this.settings = settings
+ if (getMaster() == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true")
createInterpreter()
// sets in to some kind of reader depending on environmental cues
@@ -937,16 +938,9 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter,
def createSparkContext(): SparkContext = {
val execUri = System.getenv("SPARK_EXECUTOR_URI")
- val master = this.master match {
- case Some(m) => m
- case None => {
- val prop = System.getenv("MASTER")
- if (prop != null) prop else "local"
- }
- }
val jars = SparkILoop.getAddedJars.map(new java.io.File(_).getAbsolutePath)
val conf = new SparkConf()
- .setMaster(master)
+ .setMaster(getMaster())
.setAppName("Spark shell")
.setJars(jars)
.set("spark.repl.class.uri", intp.classServer.uri)
@@ -961,6 +955,17 @@ class SparkILoop(in0: Option[BufferedReader], protected val out: JPrintWriter,
sparkContext
}
+ private def getMaster(): String = {
+ val master = this.master match {
+ case Some(m) => m
+ case None => {
+ val prop = System.getenv("MASTER")
+ if (prop != null) prop else "local"
+ }
+ }
+ master
+ }
+
/** process command-line arguments and do as they request */
def process(args: Array[String]): Boolean = {
val command = new SparkCommandLine(args.toList, msg => echo(msg))