You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/10/24 03:03:12 UTC
[1/2] git commit: Add unpersist() to JavaDoubleRDD and JavaPairRDD.
Updated Branches:
refs/heads/master dadfc63b0 -> a098438c4
Add unpersist() to JavaDoubleRDD and JavaPairRDD.
Also add support for new optional `blocking` argument.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/210858ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/210858ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/210858ac
Branch: refs/heads/master
Commit: 210858ac02350567666f6534ba62bf7978972814
Parents: 452aa36
Author: Josh Rosen <jo...@apache.org>
Authored: Wed Oct 23 14:30:29 2013 -0700
Committer: Josh Rosen <jo...@apache.org>
Committed: Wed Oct 23 17:27:01 2013 -0700
----------------------------------------------------------------------
.../org/apache/spark/api/java/JavaDoubleRDD.scala | 13 +++++++++++++
.../scala/org/apache/spark/api/java/JavaPairRDD.scala | 13 +++++++++++++
.../main/scala/org/apache/spark/api/java/JavaRDD.scala | 8 ++++++++
3 files changed, 34 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/210858ac/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
index 5fd1fab..f9b6ee3 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
@@ -48,6 +48,19 @@ class JavaDoubleRDD(val srdd: RDD[scala.Double]) extends JavaRDDLike[Double, Jav
*/
def persist(newLevel: StorageLevel): JavaDoubleRDD = fromRDD(srdd.persist(newLevel))
+ /**
+ * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+ * This method blocks until all blocks are deleted.
+ */
+ def unpersist(): JavaDoubleRDD = fromRDD(srdd.unpersist())
+
+ /**
+ * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+ *
+ * @param blocking Whether to block until all blocks are deleted.
+ */
+ def unpersist(blocking: Boolean): JavaDoubleRDD = fromRDD(srdd.unpersist(blocking))
+
// first() has to be overriden here in order for its return type to be Double instead of Object.
override def first(): Double = srdd.first()
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/210858ac/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
index a6518ab..268f43b 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
@@ -65,6 +65,19 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])(implicit val kManifest: ClassManif
def persist(newLevel: StorageLevel): JavaPairRDD[K, V] =
new JavaPairRDD[K, V](rdd.persist(newLevel))
+ /**
+ * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+ * This method blocks until all blocks are deleted.
+ */
+ def unpersist(): JavaPairRDD[K, V] = wrapRDD(rdd.unpersist())
+
+ /**
+ * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+ *
+ * @param blocking Whether to block until all blocks are deleted.
+ */
+ def unpersist(blocking: Boolean): JavaPairRDD[K, V] = wrapRDD(rdd.unpersist(blocking))
+
// Transformations (return a new RDD)
/**
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/210858ac/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
index eec58ab..6629900 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
@@ -41,9 +41,17 @@ JavaRDDLike[T, JavaRDD[T]] {
/**
* Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+ * This method blocks until all blocks are deleted.
*/
def unpersist(): JavaRDD[T] = wrapRDD(rdd.unpersist())
+ /**
+ * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+ *
+ * @param blocking Whether to block until all blocks are deleted.
+ */
+ def unpersist(blocking: Boolean): JavaRDD[T] = wrapRDD(rdd.unpersist(blocking))
+
// Transformations (return a new RDD)
/**
[2/2] git commit: Merge pull request #103 from JoshRosen/unpersist-fix
Posted by rx...@apache.org.
Merge pull request #103 from JoshRosen/unpersist-fix
Add unpersist() to JavaDoubleRDD and JavaPairRDD.
This fixes a minor inconsistency where [unpersist() was only available on JavaRDD](https://mail-archives.apache.org/mod_mbox/incubator-spark-user/201310.mbox/%3CCE8D8748.68C0%25YannLuppo%40livenation.com%3E) and not JavaPairRDD / JavaDoubleRDD. I also added support for the new optional `blocking` argument added in 0.8.
Please merge this into branch-0.8, too.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/a098438c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/a098438c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/a098438c
Branch: refs/heads/master
Commit: a098438c4824891fb44e9119e038e6e796c69741
Parents: dadfc63 210858a
Author: Reynold Xin <rx...@apache.org>
Authored: Wed Oct 23 18:03:08 2013 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Wed Oct 23 18:03:08 2013 -0700
----------------------------------------------------------------------
.../org/apache/spark/api/java/JavaDoubleRDD.scala | 13 +++++++++++++
.../scala/org/apache/spark/api/java/JavaPairRDD.scala | 13 +++++++++++++
.../main/scala/org/apache/spark/api/java/JavaRDD.scala | 8 ++++++++
3 files changed, 34 insertions(+)
----------------------------------------------------------------------