You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/10/24 03:03:12 UTC

[1/2] git commit: Add unpersist() to JavaDoubleRDD and JavaPairRDD.

Updated Branches:
  refs/heads/master dadfc63b0 -> a098438c4


Add unpersist() to JavaDoubleRDD and JavaPairRDD.

Also add support for new optional `blocking` argument.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/210858ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/210858ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/210858ac

Branch: refs/heads/master
Commit: 210858ac02350567666f6534ba62bf7978972814
Parents: 452aa36
Author: Josh Rosen <jo...@apache.org>
Authored: Wed Oct 23 14:30:29 2013 -0700
Committer: Josh Rosen <jo...@apache.org>
Committed: Wed Oct 23 17:27:01 2013 -0700

----------------------------------------------------------------------
 .../org/apache/spark/api/java/JavaDoubleRDD.scala      | 13 +++++++++++++
 .../scala/org/apache/spark/api/java/JavaPairRDD.scala  | 13 +++++++++++++
 .../main/scala/org/apache/spark/api/java/JavaRDD.scala |  8 ++++++++
 3 files changed, 34 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/210858ac/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
index 5fd1fab..f9b6ee3 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaDoubleRDD.scala
@@ -48,6 +48,19 @@ class JavaDoubleRDD(val srdd: RDD[scala.Double]) extends JavaRDDLike[Double, Jav
    */
   def persist(newLevel: StorageLevel): JavaDoubleRDD = fromRDD(srdd.persist(newLevel))
 
+  /**
+   * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+   * This method blocks until all blocks are deleted.
+   */
+  def unpersist(): JavaDoubleRDD = fromRDD(srdd.unpersist())
+
+  /**
+   * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+   *
+   * @param blocking Whether to block until all blocks are deleted.
+   */
+  def unpersist(blocking: Boolean): JavaDoubleRDD = fromRDD(srdd.unpersist(blocking))
+
   // first() has to be overriden here in order for its return type to be Double instead of Object.
   override def first(): Double = srdd.first()
 

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/210858ac/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
index a6518ab..268f43b 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaPairRDD.scala
@@ -65,6 +65,19 @@ class JavaPairRDD[K, V](val rdd: RDD[(K, V)])(implicit val kManifest: ClassManif
   def persist(newLevel: StorageLevel): JavaPairRDD[K, V] =
     new JavaPairRDD[K, V](rdd.persist(newLevel))
 
+  /**
+   * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+   * This method blocks until all blocks are deleted.
+   */
+  def unpersist(): JavaPairRDD[K, V] = wrapRDD(rdd.unpersist())
+
+  /**
+   * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+   *
+   * @param blocking Whether to block until all blocks are deleted.
+   */
+  def unpersist(blocking: Boolean): JavaPairRDD[K, V] = wrapRDD(rdd.unpersist(blocking))
+
   // Transformations (return a new RDD)
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/210858ac/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
index eec58ab..6629900 100644
--- a/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
+++ b/core/src/main/scala/org/apache/spark/api/java/JavaRDD.scala
@@ -41,9 +41,17 @@ JavaRDDLike[T, JavaRDD[T]] {
 
   /**
    * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+   * This method blocks until all blocks are deleted.
    */
   def unpersist(): JavaRDD[T] = wrapRDD(rdd.unpersist())
 
+  /**
+   * Mark the RDD as non-persistent, and remove all blocks for it from memory and disk.
+   *
+   * @param blocking Whether to block until all blocks are deleted.
+   */
+  def unpersist(blocking: Boolean): JavaRDD[T] = wrapRDD(rdd.unpersist(blocking))
+
   // Transformations (return a new RDD)
 
   /**


[2/2] git commit: Merge pull request #103 from JoshRosen/unpersist-fix

Posted by rx...@apache.org.
Merge pull request #103 from JoshRosen/unpersist-fix

Add unpersist() to JavaDoubleRDD and JavaPairRDD.

This fixes a minor inconsistency where [unpersist() was only available on JavaRDD](https://mail-archives.apache.org/mod_mbox/incubator-spark-user/201310.mbox/%3CCE8D8748.68C0%25YannLuppo%40livenation.com%3E) and not JavaPairRDD / JavaDoubleRDD.   I also added support for the new optional `blocking` argument added in 0.8.

Please merge this into branch-0.8, too.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/a098438c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/a098438c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/a098438c

Branch: refs/heads/master
Commit: a098438c4824891fb44e9119e038e6e796c69741
Parents: dadfc63 210858a
Author: Reynold Xin <rx...@apache.org>
Authored: Wed Oct 23 18:03:08 2013 -0700
Committer: Reynold Xin <rx...@apache.org>
Committed: Wed Oct 23 18:03:08 2013 -0700

----------------------------------------------------------------------
 .../org/apache/spark/api/java/JavaDoubleRDD.scala      | 13 +++++++++++++
 .../scala/org/apache/spark/api/java/JavaPairRDD.scala  | 13 +++++++++++++
 .../main/scala/org/apache/spark/api/java/JavaRDD.scala |  8 ++++++++
 3 files changed, 34 insertions(+)
----------------------------------------------------------------------