You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2013/10/31 00:58:34 UTC
[5/5] git commit: Merge pull request #124 from
tgravescs/sparkHadoopUtilFix
Merge pull request #124 from tgravescs/sparkHadoopUtilFix
Pull SparkHadoopUtil out of SparkEnv (jira SPARK-886)
Having the logic to initialize the correct SparkHadoopUtil in SparkEnv prevents it from being used until after the SparkContext is initialized. This causes issues like https://spark-project.atlassian.net/browse/SPARK-886. It also makes it hard to use in singleton objects. For instance I want to use it in the security code.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/33de11c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/33de11c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/33de11c5
Branch: refs/heads/master
Commit: 33de11c51dd2dbcbbf1801c54d9ce5ffaa324657
Parents: 618c1f6 f231aaa
Author: Matei Zaharia <ma...@eecs.berkeley.edu>
Authored: Wed Oct 30 16:58:27 2013 -0700
Committer: Matei Zaharia <ma...@eecs.berkeley.edu>
Committed: Wed Oct 30 16:58:27 2013 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/SparkContext.scala | 11 ++++-----
.../main/scala/org/apache/spark/SparkEnv.scala | 17 +++----------
.../apache/spark/deploy/SparkHadoopUtil.scala | 26 +++++++++++++++-----
.../org/apache/spark/rdd/CheckpointRDD.scala | 7 +++---
.../scala/org/apache/spark/rdd/HadoopRDD.scala | 7 +++---
.../spark/scheduler/InputFormatInfo.scala | 7 +++---
.../scala/org/apache/spark/util/Utils.scala | 3 +--
.../org/apache/spark/examples/SparkHdfsLR.scala | 3 ++-
8 files changed, 43 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/33de11c5/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --cc core/src/main/scala/org/apache/spark/SparkContext.scala
index 158197a,c1d74dc..ade75e2
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@@ -696,9 -696,8 +696,9 @@@ class SparkContext
} else {
val uri = new URI(path)
key = uri.getScheme match {
+ // A JAR file which exists only on the driver node
case null | "file" =>
- if (env.hadoop.isYarnMode()) {
+ if (SparkHadoopUtil.get.isYarnMode()) {
// In order for this to work on yarn the user must specify the --addjars option to
// the client to upload the file into the distributed cache to make it show up in the
// current working directory.