You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2014/07/30 20:00:16 UTC
git commit: [SPARK-2544][MLLIB] Improve ALS algorithm resource usage
Repository: spark
Updated Branches:
refs/heads/master e3d85b7e4 -> fc47bb696
[SPARK-2544][MLLIB] Improve ALS algorithm resource usage
Author: GuoQiang Li <wi...@qq.com>
Author: witgo <wi...@qq.com>
Closes #929 from witgo/improve_als and squashes the following commits:
ea25033 [GuoQiang Li] checkpoint products 3,6,9 ...
154dccf [GuoQiang Li] checkpoint products only
c5779ff [witgo] Improve ALS algorithm resource usage
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fc47bb69
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fc47bb69
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fc47bb69
Branch: refs/heads/master
Commit: fc47bb6967e0df40870413e09d37aa9b90248f43
Parents: e3d85b7
Author: GuoQiang Li <wi...@qq.com>
Authored: Wed Jul 30 11:00:11 2014 -0700
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Wed Jul 30 11:00:11 2014 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/mllib/recommendation/ALS.scala | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fc47bb69/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
index 5356790..d208cfb 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/ALS.scala
@@ -255,6 +255,9 @@ class ALS private (
rank, lambda, alpha, YtY)
previousProducts.unpersist()
logInfo("Re-computing U given I (Iteration %d/%d)".format(iter, iterations))
+ if (sc.checkpointDir.isDefined && (iter % 3 == 0)) {
+ products.checkpoint()
+ }
products.setName(s"products-$iter").persist()
val XtX = Some(sc.broadcast(computeYtY(products)))
val previousUsers = users
@@ -268,6 +271,9 @@ class ALS private (
logInfo("Re-computing I given U (Iteration %d/%d)".format(iter, iterations))
products = updateFeatures(numProductBlocks, users, userOutLinks, productInLinks,
rank, lambda, alpha, YtY = None)
+ if (sc.checkpointDir.isDefined && (iter % 3 == 0)) {
+ products.checkpoint()
+ }
products.setName(s"products-$iter")
logInfo("Re-computing U given I (Iteration %d/%d)".format(iter, iterations))
users = updateFeatures(numUserBlocks, products, productOutLinks, userInLinks,