You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2014/11/25 10:57:39 UTC
spark git commit: [SPARK-4596][MLLib] Refactorize Normalizer to make
code cleaner
Repository: spark
Updated Branches:
refs/heads/master 0fe54cff1 -> 89f912264
[SPARK-4596][MLLib] Refactorize Normalizer to make code cleaner
In this refactoring, the performance will be slightly increased due to removing
the overhead from breeze vector. The bottleneck is still in breeze norm
which is implemented by activeIterator.
This inefficiency of breeze norm will be addressed in next PR. At least,
this PR makes the code more consistent in the codebase.
Author: DB Tsai <db...@alpinenow.com>
Closes #3446 from dbtsai/normalizer and squashes the following commits:
e20a2b9 [DB Tsai] first commit
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/89f91226
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/89f91226
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/89f91226
Branch: refs/heads/master
Commit: 89f912264603741c7d980135c26102d63e11791f
Parents: 0fe54cf
Author: DB Tsai <db...@alpinenow.com>
Authored: Tue Nov 25 01:57:34 2014 -0800
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Tue Nov 25 01:57:34 2014 -0800
----------------------------------------------------------------------
.../apache/spark/mllib/feature/Normalizer.scala | 29 +++++++++++++-------
1 file changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/89f91226/mllib/src/main/scala/org/apache/spark/mllib/feature/Normalizer.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/feature/Normalizer.scala b/mllib/src/main/scala/org/apache/spark/mllib/feature/Normalizer.scala
index dfad25d..a9c2e23 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/feature/Normalizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/feature/Normalizer.scala
@@ -17,10 +17,10 @@
package org.apache.spark.mllib.feature
-import breeze.linalg.{DenseVector => BDV, SparseVector => BSV, norm => brzNorm}
+import breeze.linalg.{norm => brzNorm}
import org.apache.spark.annotation.Experimental
-import org.apache.spark.mllib.linalg.{Vector, Vectors}
+import org.apache.spark.mllib.linalg.{DenseVector, SparseVector, Vector, Vectors}
/**
* :: Experimental ::
@@ -47,22 +47,31 @@ class Normalizer(p: Double) extends VectorTransformer {
* @return normalized vector. If the norm of the input is zero, it will return the input vector.
*/
override def transform(vector: Vector): Vector = {
- var norm = brzNorm(vector.toBreeze, p)
+ val norm = brzNorm(vector.toBreeze, p)
if (norm != 0.0) {
// For dense vector, we've to allocate new memory for new output vector.
// However, for sparse vector, the `index` array will not be changed,
// so we can re-use it to save memory.
- vector.toBreeze match {
- case dv: BDV[Double] => Vectors.fromBreeze(dv :/ norm)
- case sv: BSV[Double] =>
- val output = new BSV[Double](sv.index, sv.data.clone(), sv.length)
+ vector match {
+ case dv: DenseVector =>
+ val values = dv.values.clone()
+ val size = values.size
var i = 0
- while (i < output.data.length) {
- output.data(i) /= norm
+ while (i < size) {
+ values(i) /= norm
i += 1
}
- Vectors.fromBreeze(output)
+ Vectors.dense(values)
+ case sv: SparseVector =>
+ val values = sv.values.clone()
+ val nnz = values.size
+ var i = 0
+ while (i < nnz) {
+ values(i) /= norm
+ i += 1
+ }
+ Vectors.sparse(sv.size, sv.indices, values)
case v => throw new IllegalArgumentException("Do not support vector type " + v.getClass)
}
} else {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org