You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2014/12/30 23:19:15 UTC
spark git commit: [Spark-4995] Replace Vector.toBreeze.activeIterator
with foreachActive
Repository: spark
Updated Branches:
refs/heads/master b239ea1c3 -> 0f31992c6
[Spark-4995] Replace Vector.toBreeze.activeIterator with foreachActive
New foreachActive method of vector was introduced by SPARK-4431 as more efficient alternative to vector.toBreeze.activeIterator. There are some parts of codebase where it was not yet replaced.
dbtsai
Author: Jakub Dubovsky <du...@avast.com>
Closes #3846 from james64/SPARK-4995-foreachActive and squashes the following commits:
3eb7e37 [Jakub Dubovsky] Scalastyle fix
32fe6c6 [Jakub Dubovsky] activeIterator removed - IndexedRowMatrix.toBreeze
47a4777 [Jakub Dubovsky] activeIterator removed in RowMatrix.toBreeze
90a7d98 [Jakub Dubovsky] activeIterator removed in MLUtils.saveAsLibSVMFile
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0f31992c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0f31992c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0f31992c
Branch: refs/heads/master
Commit: 0f31992c61f6662e5347745f6a1ac272a5fd63c9
Parents: b239ea1
Author: Jakub Dubovsky <du...@avast.com>
Authored: Tue Dec 30 14:19:07 2014 -0800
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Tue Dec 30 14:19:07 2014 -0800
----------------------------------------------------------------------
.../spark/mllib/linalg/distributed/IndexedRowMatrix.scala | 2 +-
.../apache/spark/mllib/linalg/distributed/RowMatrix.scala | 4 ++--
.../src/main/scala/org/apache/spark/mllib/util/MLUtils.scala | 8 +++++---
3 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0f31992c/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
index 5c1acca..36d8cad 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/IndexedRowMatrix.scala
@@ -142,7 +142,7 @@ class IndexedRowMatrix(
val mat = BDM.zeros[Double](m, n)
rows.collect().foreach { case IndexedRow(rowIndex, vector) =>
val i = rowIndex.toInt
- vector.toBreeze.activeIterator.foreach { case (j, v) =>
+ vector.foreachActive { case (j, v) =>
mat(i, j) = v
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/0f31992c/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index 10a515a..a3fca53 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -588,8 +588,8 @@ class RowMatrix(
val n = numCols().toInt
val mat = BDM.zeros[Double](m, n)
var i = 0
- rows.collect().foreach { v =>
- v.toBreeze.activeIterator.foreach { case (j, v) =>
+ rows.collect().foreach { vector =>
+ vector.foreachActive { case (j, v) =>
mat(i, j) = v
}
i += 1
http://git-wip-us.apache.org/repos/asf/spark/blob/0f31992c/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
index 1d07b5d..da0da0a 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MLUtils.scala
@@ -154,10 +154,12 @@ object MLUtils {
def saveAsLibSVMFile(data: RDD[LabeledPoint], dir: String) {
// TODO: allow to specify label precision and feature precision.
val dataStr = data.map { case LabeledPoint(label, features) =>
- val featureStrings = features.toBreeze.activeIterator.map { case (i, v) =>
- s"${i + 1}:$v"
+ val sb = new StringBuilder(label.toString)
+ features.foreachActive { case (i, v) =>
+ sb += ' '
+ sb ++= s"${i + 1}:$v"
}
- (Iterator(label) ++ featureStrings).mkString(" ")
+ sb.mkString
}
dataStr.saveAsTextFile(dir)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org