You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ml...@apache.org on 2017/02/22 14:36:23 UTC
spark git commit: [SPARK-19679][ML] Destroy broadcasted object
without blocking
Repository: spark
Updated Branches:
refs/heads/master ef3c73535 -> bf7bb4977
[SPARK-19679][ML] Destroy broadcasted object without blocking
## What changes were proposed in this pull request?
Destroy broadcasted object without blocking
use `find mllib -name '*.scala' | xargs -i bash -c 'egrep "destroy" -n {} && echo {}'`
## How was this patch tested?
existing tests
Author: Zheng RuiFeng <ru...@foxmail.com>
Closes #17016 from zhengruifeng/destroy_without_block.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf7bb497
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf7bb497
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf7bb497
Branch: refs/heads/master
Commit: bf7bb497784dd97de75516f300e3add478ab472d
Parents: ef3c735
Author: Zheng RuiFeng <ru...@foxmail.com>
Authored: Wed Feb 22 16:36:03 2017 +0200
Committer: Nick Pentreath <ni...@za.ibm.com>
Committed: Wed Feb 22 16:36:03 2017 +0200
----------------------------------------------------------------------
.../scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala | 2 +-
.../src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala | 2 +-
.../org/apache/spark/mllib/tree/model/treeEnsembleModels.scala | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bf7bb497/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
index f3bace8..4c525c0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/GradientBoostedTrees.scala
@@ -226,7 +226,7 @@ private[spark] object GradientBoostedTrees extends Logging {
(a, b) => treesIndices.map(idx => a(idx) + b(idx)))
.map(_ / dataCount)
- broadcastTrees.destroy()
+ broadcastTrees.destroy(blocking = false)
evaluation.toArray
}
http://git-wip-us.apache.org/repos/asf/spark/blob/bf7bb497/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
index 7a714db..efedebe 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/optimization/LBFGS.scala
@@ -261,7 +261,7 @@ object LBFGS extends Logging {
val (gradientSum, lossSum) = data.treeAggregate((zeroSparseVector, 0.0))(seqOp, combOp)
// broadcasted model is not needed anymore
- bcW.destroy()
+ bcW.destroy(blocking = false)
/**
* regVal is sum of weight squares if it's L2 updater;
http://git-wip-us.apache.org/repos/asf/spark/blob/bf7bb497/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
index fc1d412..b1e8265 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/treeEnsembleModels.scala
@@ -167,7 +167,7 @@ class GradientBoostedTreesModel @Since("1.2.0") (
(a, b) => treesIndices.map(idx => a(idx) + b(idx)))
.map(_ / dataCount)
- broadcastTrees.destroy()
+ broadcastTrees.destroy(blocking = false)
evaluation.toArray
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org