You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2014/12/08 04:42:16 UTC
spark git commit: [SPARK-4620] Add unpersist in Graph and GraphImpl
Repository: spark
Updated Branches:
refs/heads/master 2e6b736b0 -> 8817fc7fe
[SPARK-4620] Add unpersist in Graph and GraphImpl
Add an IF to uncache both vertices and edges of Graph/GraphImpl.
This IF is useful when iterative graph operations build a new graph in each iteration, and the vertices and edges of previous iterations are no longer needed for following iterations.
Author: Takeshi Yamamuro <li...@gmail.com>
This patch had conflicts when merged, resolved by
Committer: Ankur Dave <an...@gmail.com>
Closes #3476 from maropu/UnpersistInGraphSpike and squashes the following commits:
77a006a [Takeshi Yamamuro] Add unpersist in Graph and GraphImpl
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8817fc7f
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8817fc7f
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8817fc7f
Branch: refs/heads/master
Commit: 8817fc7fe8785d7b11138ca744f22f7e70f1f0a0
Parents: 2e6b736
Author: Takeshi Yamamuro <li...@gmail.com>
Authored: Sun Dec 7 19:42:02 2014 -0800
Committer: Ankur Dave <an...@gmail.com>
Committed: Sun Dec 7 19:42:02 2014 -0800
----------------------------------------------------------------------
graphx/src/main/scala/org/apache/spark/graphx/Graph.scala | 6 ++++++
.../main/scala/org/apache/spark/graphx/impl/GraphImpl.scala | 6 ++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8817fc7f/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala b/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
index 23538b7..84b72b3 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/Graph.scala
@@ -105,6 +105,12 @@ abstract class Graph[VD: ClassTag, ED: ClassTag] protected () extends Serializab
def checkpoint(): Unit
/**
+ * Uncaches both vertices and edges of this graph. This is useful in iterative algorithms that
+ * build a new graph in each iteration.
+ */
+ def unpersist(blocking: Boolean = true): Graph[VD, ED]
+
+ /**
* Uncaches only the vertices of this graph, leaving the edges alone. This is useful in iterative
* algorithms that modify the vertex attributes but reuse the edges. This method can be used to
* uncache the vertex attributes of previous iterations once they are no longer needed, improving
http://git-wip-us.apache.org/repos/asf/spark/blob/8817fc7f/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
index a617d84..3f4a900 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/impl/GraphImpl.scala
@@ -70,6 +70,12 @@ class GraphImpl[VD: ClassTag, ED: ClassTag] protected (
replicatedVertexView.edges.checkpoint()
}
+ override def unpersist(blocking: Boolean = true): Graph[VD, ED] = {
+ unpersistVertices(blocking)
+ replicatedVertexView.edges.unpersist(blocking)
+ this
+ }
+
override def unpersistVertices(blocking: Boolean = true): Graph[VD, ED] = {
vertices.unpersist(blocking)
// TODO: unpersist the replicated vertices in `replicatedVertexView` but leave the edges alone
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org