You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/19 01:24:00 UTC
[1/5] git commit: Expose method and class - so that we can use it
from user code (particularly since checkpoint directory is autogenerated now
Updated Branches:
refs/heads/master 4c16f79ce -> 73dfd42fb
Expose method and class - so that we can use it from user code (particularly since checkpoint directory is autogenerated now
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/0aea33d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/0aea33d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/0aea33d3
Branch: refs/heads/master
Commit: 0aea33d39e6ae282492f069f06b1a27a16ca0f8f
Parents: 3a386e2
Author: Mridul Muralidharan <mr...@gmail.com>
Authored: Wed Jan 15 12:44:44 2014 +0530
Committer: Mridul Muralidharan <mr...@gmail.com>
Committed: Wed Jan 15 12:44:44 2014 +0530
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 4 +++-
.../src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala | 1 -
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0aea33d3/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index ba3e91e..588e25c 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -956,6 +956,8 @@ class SparkContext(
}
}
+ def getCheckpointDir = checkpointDir.getOrElse(null)
+
/** Default level of parallelism to use when not given by user (e.g. parallelize and makeRDD). */
def defaultParallelism: Int = taskScheduler.defaultParallelism
@@ -1125,7 +1127,7 @@ object SparkContext {
if (sparkHome != null) {
res.setSparkHome(sparkHome)
}
- if (!jars.isEmpty) {
+ if (null != jars && !jars.isEmpty) {
res.setJars(jars)
}
res.setExecutorEnv(environment.toSeq)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/0aea33d3/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
index 27dc42b..b479225 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala
@@ -28,7 +28,6 @@ import org.apache.spark.{SparkContext, SparkException}
/**
* Contains util methods to interact with Hadoop from Spark.
*/
-private[spark]
class SparkHadoopUtil {
val conf = newConfiguration()
UserGroupInformation.setConfiguration(conf)
[3/5] git commit: Use method, not variable
Posted by pw...@apache.org.
Use method, not variable
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/edd82c58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/edd82c58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/edd82c58
Branch: refs/heads/master
Commit: edd82c58a22924c91f11ebadf00bdca3cff16c79
Parents: 1a0da89
Author: Mridul Muralidharan <mr...@gmail.com>
Authored: Thu Jan 16 17:26:42 2014 +0530
Committer: Mridul Muralidharan <mr...@gmail.com>
Committed: Thu Jan 16 17:26:42 2014 +0530
----------------------------------------------------------------------
.../main/scala/org/apache/spark/api/java/JavaSparkContext.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/edd82c58/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 5bd5bfd..c877870 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -401,7 +401,7 @@ class JavaSparkContext(val sc: SparkContext) extends JavaSparkContextVarargsWork
sc.setCheckpointDir(dir)
}
- def getCheckpointDir = sc.checkpointDir.getOrElse(null)
+ def getCheckpointDir = sc.getCheckpointDir.getOrElse(null)
protected def checkpointFile[T](path: String): JavaRDD[T] = {
implicit val cm: ClassTag[T] =
[4/5] git commit: Address review comment
Posted by pw...@apache.org.
Address review comment
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/b690e11d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/b690e11d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/b690e11d
Branch: refs/heads/master
Commit: b690e11d9c945c7293015db8c96f7ba0da33ec0c
Parents: edd82c5
Author: Mridul Muralidharan <mr...@gmail.com>
Authored: Fri Jan 17 18:28:55 2014 +0530
Committer: Mridul Muralidharan <mr...@gmail.com>
Committed: Fri Jan 17 18:28:55 2014 +0530
----------------------------------------------------------------------
.../main/scala/org/apache/spark/api/java/JavaSparkContext.scala | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/b690e11d/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index c877870..33c931b 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -401,7 +401,7 @@ class JavaSparkContext(val sc: SparkContext) extends JavaSparkContextVarargsWork
sc.setCheckpointDir(dir)
}
- def getCheckpointDir = sc.getCheckpointDir.getOrElse(null)
+ def getCheckpointDir = JavaUtils.optionToOptional(sc.getCheckpointDir)
protected def checkpointFile[T](path: String): JavaRDD[T] = {
implicit val cm: ClassTag[T] =
[5/5] git commit: Merge pull request #437 from mridulm/master
Posted by pw...@apache.org.
Merge pull request #437 from mridulm/master
Minor api usability changes
- Expose checkpoint directory - since it is autogenerated now
- null check for jars
- Expose SparkHadoopUtil : so that configuration creation is abstracted even from user code to avoid duplication of functionality already in spark.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/73dfd42f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/73dfd42f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/73dfd42f
Branch: refs/heads/master
Commit: 73dfd42fba5e526cc57e2a2ed78be323b63cb8fa
Parents: 4c16f79 b690e11
Author: Patrick Wendell <pw...@gmail.com>
Authored: Sat Jan 18 16:23:56 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Sat Jan 18 16:23:56 2014 -0800
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 4 +++-
.../main/scala/org/apache/spark/api/java/JavaSparkContext.scala | 2 ++
.../src/main/scala/org/apache/spark/deploy/SparkHadoopUtil.scala | 1 -
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/73dfd42f/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
[2/5] git commit: Address review comments
Posted by pw...@apache.org.
Address review comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/1a0da892
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/1a0da892
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/1a0da892
Branch: refs/heads/master
Commit: 1a0da892771e6138dd449c84fc64e4c567640939
Parents: 0aea33d
Author: Mridul Muralidharan <mr...@gmail.com>
Authored: Thu Jan 16 17:23:25 2014 +0530
Committer: Mridul Muralidharan <mr...@gmail.com>
Committed: Thu Jan 16 17:23:25 2014 +0530
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/SparkContext.scala | 4 ++--
.../main/scala/org/apache/spark/api/java/JavaSparkContext.scala | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/1a0da892/core/src/main/scala/org/apache/spark/SparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/SparkContext.scala b/core/src/main/scala/org/apache/spark/SparkContext.scala
index 588e25c..ddd7d60 100644
--- a/core/src/main/scala/org/apache/spark/SparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/SparkContext.scala
@@ -956,7 +956,7 @@ class SparkContext(
}
}
- def getCheckpointDir = checkpointDir.getOrElse(null)
+ def getCheckpointDir = checkpointDir
/** Default level of parallelism to use when not given by user (e.g. parallelize and makeRDD). */
def defaultParallelism: Int = taskScheduler.defaultParallelism
@@ -1127,7 +1127,7 @@ object SparkContext {
if (sparkHome != null) {
res.setSparkHome(sparkHome)
}
- if (null != jars && !jars.isEmpty) {
+ if (jars != null && !jars.isEmpty) {
res.setJars(jars)
}
res.setExecutorEnv(environment.toSeq)
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/1a0da892/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
index 8041163..5bd5bfd 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaSparkContext.scala
@@ -401,6 +401,8 @@ class JavaSparkContext(val sc: SparkContext) extends JavaSparkContextVarargsWork
sc.setCheckpointDir(dir)
}
+ def getCheckpointDir = sc.checkpointDir.getOrElse(null)
+
protected def checkpointFile[T](path: String): JavaRDD[T] = {
implicit val cm: ClassTag[T] =
implicitly[ClassTag[AnyRef]].asInstanceOf[ClassTag[T]]