You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/07/25 20:43:42 UTC
spark git commit: [SPARK-21491][GRAPHX] Enhance GraphX performance:
breakOut instead of .toMap
Repository: spark
Updated Branches:
refs/heads/master 06a979379 -> 9b4da7b79
[SPARK-21491][GRAPHX] Enhance GraphX performance: breakOut instead of .toMap
## What changes were proposed in this pull request?
`Traversable.toMap` changed to 'collections.breakOut', that eliminates intermediate tuple collection creation, see [Stack Overflow article](https://stackoverflow.com/questions/1715681/scala-2-8-breakout).
## How was this patch tested?
Unit tests run.
No performance tests performed yet.
Please review http://spark.apache.org/contributing.html before opening a pull request.
Author: iurii.ant <se...@gmail.com>
Closes #18693 from SereneAnt/performance_toMap-breakOut.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9b4da7b7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9b4da7b7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9b4da7b7
Branch: refs/heads/master
Commit: 9b4da7b7906131e42fcac109d40e4d663f57c291
Parents: 06a9793
Author: iurii.ant <se...@gmail.com>
Authored: Tue Jul 25 21:43:39 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Jul 25 21:43:39 2017 +0100
----------------------------------------------------------------------
.../scala/org/apache/spark/graphx/lib/LabelPropagation.scala | 2 +-
.../main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala | 5 +++--
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9b4da7b7/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
index fc7547a..cb3025f 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/LabelPropagation.scala
@@ -55,7 +55,7 @@ object LabelPropagation {
val count1Val = count1.getOrElse(i, 0L)
val count2Val = count2.getOrElse(i, 0L)
i -> (count1Val + count2Val)
- }.toMap
+ }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]]
}
def vertexProgram(vid: VertexId, attr: Long, message: Map[VertexId, Long]): VertexId = {
if (message.isEmpty) attr else message.maxBy(_._2)._1
http://git-wip-us.apache.org/repos/asf/spark/blob/9b4da7b7/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
----------------------------------------------------------------------
diff --git a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
index f0c6bcb..4cac633 100644
--- a/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
+++ b/graphx/src/main/scala/org/apache/spark/graphx/lib/ShortestPaths.scala
@@ -33,10 +33,11 @@ object ShortestPaths {
private def incrementMap(spmap: SPMap): SPMap = spmap.map { case (v, d) => v -> (d + 1) }
- private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap =
+ private def addMaps(spmap1: SPMap, spmap2: SPMap): SPMap = {
(spmap1.keySet ++ spmap2.keySet).map {
k => k -> math.min(spmap1.getOrElse(k, Int.MaxValue), spmap2.getOrElse(k, Int.MaxValue))
- }.toMap
+ }(collection.breakOut) // more efficient alternative to [[collection.Traversable.toMap]]
+ }
/**
* Computes shortest paths to the given set of landmark vertices.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org