You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/08/16 18:05:24 UTC
spark git commit: [SPARK-21680][ML][MLLIB] optimize Vector compress
Repository: spark
Updated Branches:
refs/heads/master 7add4e982 -> a0345cbeb
[SPARK-21680][ML][MLLIB] optimize Vector compress
## What changes were proposed in this pull request?
When use Vector.compressed to change a Vector to SparseVector, the performance is very low comparing with Vector.toSparse.
This is because you have to scan the value three times using Vector.compressed, but you just need two times when use Vector.toSparse.
When the length of the vector is large, there is significant performance difference between this two method.
## How was this patch tested?
The existing UT
Author: Peng Meng <pe...@intel.com>
Closes #18899 from mpjlu/optVectorCompress.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a0345cbe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a0345cbe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a0345cbe
Branch: refs/heads/master
Commit: a0345cbebe23537df4084cf90f9d47425e550150
Parents: 7add4e9
Author: Peng Meng <pe...@intel.com>
Authored: Wed Aug 16 19:05:20 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed Aug 16 19:05:20 2017 +0100
----------------------------------------------------------------------
.../org/apache/spark/ml/linalg/Vectors.scala | 24 +++++++++++++-----
.../apache/spark/ml/linalg/VectorsSuite.scala | 10 ++++++++
.../org/apache/spark/mllib/linalg/Vectors.scala | 26 ++++++++++++++------
.../spark/mllib/linalg/VectorsSuite.scala | 10 ++++++++
project/MimaExcludes.scala | 4 +++
5 files changed, 60 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a0345cbe/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
index 3fbc095..941b6ec 100644
--- a/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
+++ b/mllib-local/src/main/scala/org/apache/spark/ml/linalg/Vectors.scala
@@ -136,7 +136,21 @@ sealed trait Vector extends Serializable {
* Converts this vector to a sparse vector with all explicit zeros removed.
*/
@Since("2.0.0")
- def toSparse: SparseVector
+ def toSparse: SparseVector = toSparseWithSize(numNonzeros)
+
+ /**
+ * Converts this vector to a sparse vector with all explicit zeros removed when the size is known.
+ * This method is used to avoid re-computing the number of non-zero elements when it is
+ * already known. This method should only be called after computing the number of non-zero
+ * elements via [[numNonzeros]]. e.g.
+ * {{{
+ * val nnz = numNonzeros
+ * val sv = toSparse(nnz)
+ * }}}
+ *
+ * If `nnz` is under-specified, a [[java.lang.ArrayIndexOutOfBoundsException]] is thrown.
+ */
+ private[linalg] def toSparseWithSize(nnz: Int): SparseVector
/**
* Converts this vector to a dense vector.
@@ -152,7 +166,7 @@ sealed trait Vector extends Serializable {
val nnz = numNonzeros
// A dense vector needs 8 * size + 8 bytes, while a sparse vector needs 12 * nnz + 20 bytes.
if (1.5 * (nnz + 1.0) < size) {
- toSparse
+ toSparseWithSize(nnz)
} else {
toDense
}
@@ -495,8 +509,7 @@ class DenseVector @Since("2.0.0") ( @Since("2.0.0") val values: Array[Double]) e
nnz
}
- override def toSparse: SparseVector = {
- val nnz = numNonzeros
+ private[linalg] override def toSparseWithSize(nnz: Int): SparseVector = {
val ii = new Array[Int](nnz)
val vv = new Array[Double](nnz)
var k = 0
@@ -635,8 +648,7 @@ class SparseVector @Since("2.0.0") (
nnz
}
- override def toSparse: SparseVector = {
- val nnz = numNonzeros
+ private[linalg] override def toSparseWithSize(nnz: Int): SparseVector = {
if (nnz == numActives) {
this
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/a0345cbe/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
----------------------------------------------------------------------
diff --git a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
index 4cd91af..79acef8 100644
--- a/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
+++ b/mllib-local/src/test/scala/org/apache/spark/ml/linalg/VectorsSuite.scala
@@ -318,11 +318,21 @@ class VectorsSuite extends SparkMLFunSuite {
assert(dv0s.numActives === 2)
assert(dv0s === dv0)
+ assert(dv0.toSparseWithSize(dv0.numNonzeros) === dv0)
+ val dv0s2 = dv0.toSparseWithSize(dv0.numNonzeros)
+ assert(dv0s2.numActives === 2)
+ assert(dv0s2 === dv0s)
+
val sv0 = Vectors.sparse(4, Array(0, 1, 2), Array(0.0, 2.0, 3.0))
assert(sv0.toDense === sv0)
val sv0s = sv0.toSparse
assert(sv0s.numActives === 2)
assert(sv0s === sv0)
+
+ assert(sv0.toSparseWithSize(sv0.numNonzeros) === sv0)
+ val sv0s2 = sv0.toSparseWithSize(sv0.numNonzeros)
+ assert(sv0s2.numActives === 2)
+ assert(sv0s2 === sv0s)
}
test("Vector.compressed") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a0345cbe/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
index f063420..fd9605c 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Vectors.scala
@@ -149,7 +149,21 @@ sealed trait Vector extends Serializable {
* Converts this vector to a sparse vector with all explicit zeros removed.
*/
@Since("1.4.0")
- def toSparse: SparseVector
+ def toSparse: SparseVector = toSparseWithSize(numNonzeros)
+
+ /**
+ * Converts this vector to a sparse vector with all explicit zeros removed when the size is known.
+ * This method is used to avoid re-computing the number of non-zero elements when it is
+ * already known. This method should only be called after computing the number of non-zero
+ * elements via [[numNonzeros]]. e.g.
+ * {{{
+ * val nnz = numNonzeros
+ * val sv = toSparse(nnz)
+ * }}}
+ *
+ * If `nnz` is under-specified, a [[java.lang.ArrayIndexOutOfBoundsException]] is thrown.
+ */
+ private[linalg] def toSparseWithSize(nnz: Int): SparseVector
/**
* Converts this vector to a dense vector.
@@ -165,7 +179,7 @@ sealed trait Vector extends Serializable {
val nnz = numNonzeros
// A dense vector needs 8 * size + 8 bytes, while a sparse vector needs 12 * nnz + 20 bytes.
if (1.5 * (nnz + 1.0) < size) {
- toSparse
+ toSparseWithSize(nnz)
} else {
toDense
}
@@ -669,9 +683,7 @@ class DenseVector @Since("1.0.0") (
nnz
}
- @Since("1.4.0")
- override def toSparse: SparseVector = {
- val nnz = numNonzeros
+ private[linalg] override def toSparseWithSize(nnz: Int): SparseVector = {
val ii = new Array[Int](nnz)
val vv = new Array[Double](nnz)
var k = 0
@@ -822,9 +834,7 @@ class SparseVector @Since("1.0.0") (
nnz
}
- @Since("1.4.0")
- override def toSparse: SparseVector = {
- val nnz = numNonzeros
+ private[linalg] override def toSparseWithSize(nnz: Int): SparseVector = {
if (nnz == numActives) {
this
} else {
http://git-wip-us.apache.org/repos/asf/spark/blob/a0345cbe/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
index 6172cff..a1e3ee5 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/VectorsSuite.scala
@@ -351,11 +351,21 @@ class VectorsSuite extends SparkFunSuite with Logging {
assert(dv0s.numActives === 2)
assert(dv0s === dv0)
+ assert(dv0.toSparseWithSize(dv0.numNonzeros) === dv0)
+ val dv0s2 = dv0.toSparseWithSize(dv0.numNonzeros)
+ assert(dv0s2.numActives === 2)
+ assert(dv0s2 === dv0s)
+
val sv0 = Vectors.sparse(4, Array(0, 1, 2), Array(0.0, 2.0, 3.0))
assert(sv0.toDense === sv0)
val sv0s = sv0.toSparse
assert(sv0s.numActives === 2)
assert(sv0s === sv0)
+
+ assert(sv0.toSparseWithSize(sv0.numNonzeros) === sv0)
+ val sv0s2 = sv0.toSparseWithSize(sv0.numNonzeros)
+ assert(sv0s2.numActives === 2)
+ assert(sv0s2 === sv0s)
}
test("Vector.compressed") {
http://git-wip-us.apache.org/repos/asf/spark/blob/a0345cbe/project/MimaExcludes.scala
----------------------------------------------------------------------
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 7ba85bd..9bda917 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -1015,6 +1015,10 @@ object MimaExcludes {
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.classification.RandomForestClassificationModel.setFeatureSubsetStrategy"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.numTrees"),
ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.ml.regression.RandomForestRegressionModel.setFeatureSubsetStrategy")
+ ) ++ Seq(
+ // [SPARK-21680][ML][MLLIB]optimzie Vector coompress
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.mllib.linalg.Vector.toSparseWithSize"),
+ ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.ml.linalg.Vector.toSparseWithSize")
)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org