You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/04/10 13:51:38 UTC
spark git commit: [SPARK-23841][ML] NodeIdCache should unpersist the
last cached nodeIdsForInstances
Repository: spark
Updated Branches:
refs/heads/master 649888415 -> 95034af69
[SPARK-23841][ML] NodeIdCache should unpersist the last cached nodeIdsForInstances
## What changes were proposed in this pull request?
unpersist the last cached nodeIdsForInstances in `deleteAllCheckpoints`
## How was this patch tested?
existing tests
Author: Zheng RuiFeng <ru...@foxmail.com>
Closes #20956 from zhengruifeng/NodeIdCache_cleanup.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/95034af6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/95034af6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/95034af6
Branch: refs/heads/master
Commit: 95034af69623bb8be5b9f5eabf50980bdeca48e6
Parents: 6498884
Author: Zheng RuiFeng <ru...@foxmail.com>
Authored: Tue Apr 10 08:51:35 2018 -0500
Committer: Sean Owen <sr...@gmail.com>
Committed: Tue Apr 10 08:51:35 2018 -0500
----------------------------------------------------------------------
.../scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/95034af6/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala
index a7c5f48..5b14a63 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/NodeIdCache.scala
@@ -95,7 +95,7 @@ private[spark] class NodeIdCache(
splits: Array[Array[Split]]): Unit = {
if (prevNodeIdsForInstances != null) {
// Unpersist the previous one if one exists.
- prevNodeIdsForInstances.unpersist()
+ prevNodeIdsForInstances.unpersist(false)
}
prevNodeIdsForInstances = nodeIdsForInstances
@@ -166,9 +166,13 @@ private[spark] class NodeIdCache(
}
}
}
+ if (nodeIdsForInstances != null) {
+ // Unpersist current one if one exists.
+ nodeIdsForInstances.unpersist(false)
+ }
if (prevNodeIdsForInstances != null) {
// Unpersist the previous one if one exists.
- prevNodeIdsForInstances.unpersist()
+ prevNodeIdsForInstances.unpersist(false)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org