You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2015/01/27 10:46:26 UTC
spark git commit: [SPARK-5321] Support for transposing local matrices
Repository: spark
Updated Branches:
refs/heads/master 7b0ed7979 -> 914267484
[SPARK-5321] Support for transposing local matrices
Support for transposing local matrices added. The `.transpose` function creates a new object re-using the backing array(s) but switches `numRows` and `numCols`. Operations check the flag `.isTransposed` to see whether the indexing in `values` should be modified.
This PR will pave the way for transposing `BlockMatrix`.
Author: Burak Yavuz <br...@gmail.com>
Closes #4109 from brkyvz/SPARK-5321 and squashes the following commits:
87ab83c [Burak Yavuz] fixed scalastyle
caf4438 [Burak Yavuz] addressed code review v3
c524770 [Burak Yavuz] address code review comments 2
77481e8 [Burak Yavuz] fixed MiMa
f1c1742 [Burak Yavuz] small refactoring
ccccdec [Burak Yavuz] fixed failed test
dd45c88 [Burak Yavuz] addressed code review
a01bd5f [Burak Yavuz] [SPARK-5321] Fixed MiMa issues
2a63593 [Burak Yavuz] [SPARK-5321] fixed bug causing failed gemm test
c55f29a [Burak Yavuz] [SPARK-5321] Support for transposing local matrices cleaned up
c408c05 [Burak Yavuz] [SPARK-5321] Support for transposing local matrices added
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/91426748
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/91426748
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/91426748
Branch: refs/heads/master
Commit: 914267484a7156718ab6da37a6a42bbb074b51ac
Parents: 7b0ed79
Author: Burak Yavuz <br...@gmail.com>
Authored: Tue Jan 27 01:46:17 2015 -0800
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Tue Jan 27 01:46:17 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/mllib/linalg/BLAS.scala | 170 ++++-----
.../apache/spark/mllib/linalg/Matrices.scala | 347 ++++++++++++-------
.../apache/spark/mllib/linalg/BLASSuite.scala | 72 ++--
.../linalg/BreezeMatrixConversionSuite.scala | 9 +
.../spark/mllib/linalg/MatricesSuite.scala | 106 +++++-
project/MimaExcludes.scala | 14 +
6 files changed, 436 insertions(+), 282 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/91426748/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
index 3414dac..34e0392 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/BLAS.scala
@@ -257,30 +257,28 @@ private[spark] object BLAS extends Serializable with Logging {
/**
* C := alpha * A * B + beta * C
- * @param transA whether to use the transpose of matrix A (true), or A itself (false).
- * @param transB whether to use the transpose of matrix B (true), or B itself (false).
* @param alpha a scalar to scale the multiplication A * B.
* @param A the matrix A that will be left multiplied to B. Size of m x k.
* @param B the matrix B that will be left multiplied by A. Size of k x n.
* @param beta a scalar that can be used to scale matrix C.
- * @param C the resulting matrix C. Size of m x n.
+ * @param C the resulting matrix C. Size of m x n. C.isTransposed must be false.
*/
def gemm(
- transA: Boolean,
- transB: Boolean,
alpha: Double,
A: Matrix,
B: DenseMatrix,
beta: Double,
C: DenseMatrix): Unit = {
+ require(!C.isTransposed,
+ "The matrix C cannot be the product of a transpose() call. C.isTransposed must be false.")
if (alpha == 0.0) {
logDebug("gemm: alpha is equal to 0. Returning C.")
} else {
A match {
case sparse: SparseMatrix =>
- gemm(transA, transB, alpha, sparse, B, beta, C)
+ gemm(alpha, sparse, B, beta, C)
case dense: DenseMatrix =>
- gemm(transA, transB, alpha, dense, B, beta, C)
+ gemm(alpha, dense, B, beta, C)
case _ =>
throw new IllegalArgumentException(s"gemm doesn't support matrix type ${A.getClass}.")
}
@@ -289,48 +287,28 @@ private[spark] object BLAS extends Serializable with Logging {
/**
* C := alpha * A * B + beta * C
- *
- * @param alpha a scalar to scale the multiplication A * B.
- * @param A the matrix A that will be left multiplied to B. Size of m x k.
- * @param B the matrix B that will be left multiplied by A. Size of k x n.
- * @param beta a scalar that can be used to scale matrix C.
- * @param C the resulting matrix C. Size of m x n.
- */
- def gemm(
- alpha: Double,
- A: Matrix,
- B: DenseMatrix,
- beta: Double,
- C: DenseMatrix): Unit = {
- gemm(false, false, alpha, A, B, beta, C)
- }
-
- /**
- * C := alpha * A * B + beta * C
* For `DenseMatrix` A.
*/
private def gemm(
- transA: Boolean,
- transB: Boolean,
alpha: Double,
A: DenseMatrix,
B: DenseMatrix,
beta: Double,
C: DenseMatrix): Unit = {
- val mA: Int = if (!transA) A.numRows else A.numCols
- val nB: Int = if (!transB) B.numCols else B.numRows
- val kA: Int = if (!transA) A.numCols else A.numRows
- val kB: Int = if (!transB) B.numRows else B.numCols
- val tAstr = if (!transA) "N" else "T"
- val tBstr = if (!transB) "N" else "T"
-
- require(kA == kB, s"The columns of A don't match the rows of B. A: $kA, B: $kB")
- require(mA == C.numRows, s"The rows of C don't match the rows of A. C: ${C.numRows}, A: $mA")
- require(nB == C.numCols,
- s"The columns of C don't match the columns of B. C: ${C.numCols}, A: $nB")
-
- nativeBLAS.dgemm(tAstr, tBstr, mA, nB, kA, alpha, A.values, A.numRows, B.values, B.numRows,
- beta, C.values, C.numRows)
+ val tAstr = if (A.isTransposed) "T" else "N"
+ val tBstr = if (B.isTransposed) "T" else "N"
+ val lda = if (!A.isTransposed) A.numRows else A.numCols
+ val ldb = if (!B.isTransposed) B.numRows else B.numCols
+
+ require(A.numCols == B.numRows,
+ s"The columns of A don't match the rows of B. A: ${A.numCols}, B: ${B.numRows}")
+ require(A.numRows == C.numRows,
+ s"The rows of C don't match the rows of A. C: ${C.numRows}, A: ${A.numRows}")
+ require(B.numCols == C.numCols,
+ s"The columns of C don't match the columns of B. C: ${C.numCols}, A: ${B.numCols}")
+
+ nativeBLAS.dgemm(tAstr, tBstr, A.numRows, B.numCols, A.numCols, alpha, A.values, lda,
+ B.values, ldb, beta, C.values, C.numRows)
}
/**
@@ -338,17 +316,15 @@ private[spark] object BLAS extends Serializable with Logging {
* For `SparseMatrix` A.
*/
private def gemm(
- transA: Boolean,
- transB: Boolean,
alpha: Double,
A: SparseMatrix,
B: DenseMatrix,
beta: Double,
C: DenseMatrix): Unit = {
- val mA: Int = if (!transA) A.numRows else A.numCols
- val nB: Int = if (!transB) B.numCols else B.numRows
- val kA: Int = if (!transA) A.numCols else A.numRows
- val kB: Int = if (!transB) B.numRows else B.numCols
+ val mA: Int = A.numRows
+ val nB: Int = B.numCols
+ val kA: Int = A.numCols
+ val kB: Int = B.numRows
require(kA == kB, s"The columns of A don't match the rows of B. A: $kA, B: $kB")
require(mA == C.numRows, s"The rows of C don't match the rows of A. C: ${C.numRows}, A: $mA")
@@ -358,23 +334,23 @@ private[spark] object BLAS extends Serializable with Logging {
val Avals = A.values
val Bvals = B.values
val Cvals = C.values
- val Arows = if (!transA) A.rowIndices else A.colPtrs
- val Acols = if (!transA) A.colPtrs else A.rowIndices
+ val ArowIndices = A.rowIndices
+ val AcolPtrs = A.colPtrs
// Slicing is easy in this case. This is the optimal multiplication setting for sparse matrices
- if (transA){
+ if (A.isTransposed){
var colCounterForB = 0
- if (!transB) { // Expensive to put the check inside the loop
+ if (!B.isTransposed) { // Expensive to put the check inside the loop
while (colCounterForB < nB) {
var rowCounterForA = 0
val Cstart = colCounterForB * mA
val Bstart = colCounterForB * kA
while (rowCounterForA < mA) {
- var i = Arows(rowCounterForA)
- val indEnd = Arows(rowCounterForA + 1)
+ var i = AcolPtrs(rowCounterForA)
+ val indEnd = AcolPtrs(rowCounterForA + 1)
var sum = 0.0
while (i < indEnd) {
- sum += Avals(i) * Bvals(Bstart + Acols(i))
+ sum += Avals(i) * Bvals(Bstart + ArowIndices(i))
i += 1
}
val Cindex = Cstart + rowCounterForA
@@ -385,19 +361,19 @@ private[spark] object BLAS extends Serializable with Logging {
}
} else {
while (colCounterForB < nB) {
- var rowCounter = 0
+ var rowCounterForA = 0
val Cstart = colCounterForB * mA
- while (rowCounter < mA) {
- var i = Arows(rowCounter)
- val indEnd = Arows(rowCounter + 1)
+ while (rowCounterForA < mA) {
+ var i = AcolPtrs(rowCounterForA)
+ val indEnd = AcolPtrs(rowCounterForA + 1)
var sum = 0.0
while (i < indEnd) {
- sum += Avals(i) * B(colCounterForB, Acols(i))
+ sum += Avals(i) * B(ArowIndices(i), colCounterForB)
i += 1
}
- val Cindex = Cstart + rowCounter
+ val Cindex = Cstart + rowCounterForA
Cvals(Cindex) = beta * Cvals(Cindex) + sum * alpha
- rowCounter += 1
+ rowCounterForA += 1
}
colCounterForB += 1
}
@@ -410,17 +386,17 @@ private[spark] object BLAS extends Serializable with Logging {
// Perform matrix multiplication and add to C. The rows of A are multiplied by the columns of
// B, and added to C.
var colCounterForB = 0 // the column to be updated in C
- if (!transB) { // Expensive to put the check inside the loop
+ if (!B.isTransposed) { // Expensive to put the check inside the loop
while (colCounterForB < nB) {
var colCounterForA = 0 // The column of A to multiply with the row of B
val Bstart = colCounterForB * kB
val Cstart = colCounterForB * mA
while (colCounterForA < kA) {
- var i = Acols(colCounterForA)
- val indEnd = Acols(colCounterForA + 1)
+ var i = AcolPtrs(colCounterForA)
+ val indEnd = AcolPtrs(colCounterForA + 1)
val Bval = Bvals(Bstart + colCounterForA) * alpha
while (i < indEnd) {
- Cvals(Cstart + Arows(i)) += Avals(i) * Bval
+ Cvals(Cstart + ArowIndices(i)) += Avals(i) * Bval
i += 1
}
colCounterForA += 1
@@ -432,11 +408,11 @@ private[spark] object BLAS extends Serializable with Logging {
var colCounterForA = 0 // The column of A to multiply with the row of B
val Cstart = colCounterForB * mA
while (colCounterForA < kA) {
- var i = Acols(colCounterForA)
- val indEnd = Acols(colCounterForA + 1)
- val Bval = B(colCounterForB, colCounterForA) * alpha
+ var i = AcolPtrs(colCounterForA)
+ val indEnd = AcolPtrs(colCounterForA + 1)
+ val Bval = B(colCounterForA, colCounterForB) * alpha
while (i < indEnd) {
- Cvals(Cstart + Arows(i)) += Avals(i) * Bval
+ Cvals(Cstart + ArowIndices(i)) += Avals(i) * Bval
i += 1
}
colCounterForA += 1
@@ -449,7 +425,6 @@ private[spark] object BLAS extends Serializable with Logging {
/**
* y := alpha * A * x + beta * y
- * @param trans whether to use the transpose of matrix A (true), or A itself (false).
* @param alpha a scalar to scale the multiplication A * x.
* @param A the matrix A that will be left multiplied to x. Size of m x n.
* @param x the vector x that will be left multiplied by A. Size of n x 1.
@@ -457,28 +432,23 @@ private[spark] object BLAS extends Serializable with Logging {
* @param y the resulting vector y. Size of m x 1.
*/
def gemv(
- trans: Boolean,
alpha: Double,
A: Matrix,
x: DenseVector,
beta: Double,
y: DenseVector): Unit = {
-
- val mA: Int = if (!trans) A.numRows else A.numCols
- val nx: Int = x.size
- val nA: Int = if (!trans) A.numCols else A.numRows
-
- require(nA == nx, s"The columns of A don't match the number of elements of x. A: $nA, x: $nx")
- require(mA == y.size,
- s"The rows of A don't match the number of elements of y. A: $mA, y:${y.size}}")
+ require(A.numCols == x.size,
+ s"The columns of A don't match the number of elements of x. A: ${A.numCols}, x: ${x.size}")
+ require(A.numRows == y.size,
+ s"The rows of A don't match the number of elements of y. A: ${A.numRows}, y:${y.size}}")
if (alpha == 0.0) {
logDebug("gemv: alpha is equal to 0. Returning y.")
} else {
A match {
case sparse: SparseMatrix =>
- gemv(trans, alpha, sparse, x, beta, y)
+ gemv(alpha, sparse, x, beta, y)
case dense: DenseMatrix =>
- gemv(trans, alpha, dense, x, beta, y)
+ gemv(alpha, dense, x, beta, y)
case _ =>
throw new IllegalArgumentException(s"gemv doesn't support matrix type ${A.getClass}.")
}
@@ -487,35 +457,18 @@ private[spark] object BLAS extends Serializable with Logging {
/**
* y := alpha * A * x + beta * y
- *
- * @param alpha a scalar to scale the multiplication A * x.
- * @param A the matrix A that will be left multiplied to x. Size of m x n.
- * @param x the vector x that will be left multiplied by A. Size of n x 1.
- * @param beta a scalar that can be used to scale vector y.
- * @param y the resulting vector y. Size of m x 1.
- */
- def gemv(
- alpha: Double,
- A: Matrix,
- x: DenseVector,
- beta: Double,
- y: DenseVector): Unit = {
- gemv(false, alpha, A, x, beta, y)
- }
-
- /**
- * y := alpha * A * x + beta * y
* For `DenseMatrix` A.
*/
private def gemv(
- trans: Boolean,
alpha: Double,
A: DenseMatrix,
x: DenseVector,
beta: Double,
y: DenseVector): Unit = {
- val tStrA = if (!trans) "N" else "T"
- nativeBLAS.dgemv(tStrA, A.numRows, A.numCols, alpha, A.values, A.numRows, x.values, 1, beta,
+ val tStrA = if (A.isTransposed) "T" else "N"
+ val mA = if (!A.isTransposed) A.numRows else A.numCols
+ val nA = if (!A.isTransposed) A.numCols else A.numRows
+ nativeBLAS.dgemv(tStrA, mA, nA, alpha, A.values, mA, x.values, 1, beta,
y.values, 1)
}
@@ -524,24 +477,21 @@ private[spark] object BLAS extends Serializable with Logging {
* For `SparseMatrix` A.
*/
private def gemv(
- trans: Boolean,
alpha: Double,
A: SparseMatrix,
x: DenseVector,
beta: Double,
y: DenseVector): Unit = {
-
val xValues = x.values
val yValues = y.values
-
- val mA: Int = if (!trans) A.numRows else A.numCols
- val nA: Int = if (!trans) A.numCols else A.numRows
+ val mA: Int = A.numRows
+ val nA: Int = A.numCols
val Avals = A.values
- val Arows = if (!trans) A.rowIndices else A.colPtrs
- val Acols = if (!trans) A.colPtrs else A.rowIndices
+ val Arows = if (!A.isTransposed) A.rowIndices else A.colPtrs
+ val Acols = if (!A.isTransposed) A.colPtrs else A.rowIndices
// Slicing is easy in this case. This is the optimal multiplication setting for sparse matrices
- if (trans) {
+ if (A.isTransposed) {
var rowCounter = 0
while (rowCounter < mA) {
var i = Arows(rowCounter)
http://git-wip-us.apache.org/repos/asf/spark/blob/91426748/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
index 5a7281e..ad7e868 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/Matrices.scala
@@ -34,8 +34,17 @@ sealed trait Matrix extends Serializable {
/** Number of columns. */
def numCols: Int
+ /** Flag that keeps track whether the matrix is transposed or not. False by default. */
+ val isTransposed: Boolean = false
+
/** Converts to a dense array in column major. */
- def toArray: Array[Double]
+ def toArray: Array[Double] = {
+ val newArray = new Array[Double](numRows * numCols)
+ foreachActive { (i, j, v) =>
+ newArray(j * numRows + i) = v
+ }
+ newArray
+ }
/** Converts to a breeze matrix. */
private[mllib] def toBreeze: BM[Double]
@@ -52,10 +61,13 @@ sealed trait Matrix extends Serializable {
/** Get a deep copy of the matrix. */
def copy: Matrix
+ /** Transpose the Matrix. Returns a new `Matrix` instance sharing the same underlying data. */
+ def transpose: Matrix
+
/** Convenience method for `Matrix`-`DenseMatrix` multiplication. */
def multiply(y: DenseMatrix): DenseMatrix = {
- val C: DenseMatrix = Matrices.zeros(numRows, y.numCols).asInstanceOf[DenseMatrix]
- BLAS.gemm(false, false, 1.0, this, y, 0.0, C)
+ val C: DenseMatrix = DenseMatrix.zeros(numRows, y.numCols)
+ BLAS.gemm(1.0, this, y, 0.0, C)
C
}
@@ -66,20 +78,6 @@ sealed trait Matrix extends Serializable {
output
}
- /** Convenience method for `Matrix`^T^-`DenseMatrix` multiplication. */
- private[mllib] def transposeMultiply(y: DenseMatrix): DenseMatrix = {
- val C: DenseMatrix = Matrices.zeros(numCols, y.numCols).asInstanceOf[DenseMatrix]
- BLAS.gemm(true, false, 1.0, this, y, 0.0, C)
- C
- }
-
- /** Convenience method for `Matrix`^T^-`DenseVector` multiplication. */
- private[mllib] def transposeMultiply(y: DenseVector): DenseVector = {
- val output = new DenseVector(new Array[Double](numCols))
- BLAS.gemv(true, 1.0, this, y, 0.0, output)
- output
- }
-
/** A human readable representation of the matrix */
override def toString: String = toBreeze.toString()
@@ -92,6 +90,16 @@ sealed trait Matrix extends Serializable {
* backing array. For example, an operation such as addition or subtraction will only be
* performed on the non-zero values in a `SparseMatrix`. */
private[mllib] def update(f: Double => Double): Matrix
+
+ /**
+ * Applies a function `f` to all the active elements of dense and sparse matrix. The ordering
+ * of the elements are not defined.
+ *
+ * @param f the function takes three parameters where the first two parameters are the row
+ * and column indices respectively with the type `Int`, and the final parameter is the
+ * corresponding value in the matrix with type `Double`.
+ */
+ private[spark] def foreachActive(f: (Int, Int, Double) => Unit)
}
/**
@@ -108,13 +116,35 @@ sealed trait Matrix extends Serializable {
* @param numRows number of rows
* @param numCols number of columns
* @param values matrix entries in column major
+ * @param isTransposed whether the matrix is transposed. If true, `values` stores the matrix in
+ * row major.
*/
-class DenseMatrix(val numRows: Int, val numCols: Int, val values: Array[Double]) extends Matrix {
+class DenseMatrix(
+ val numRows: Int,
+ val numCols: Int,
+ val values: Array[Double],
+ override val isTransposed: Boolean) extends Matrix {
require(values.length == numRows * numCols, "The number of values supplied doesn't match the " +
s"size of the matrix! values.length: ${values.length}, numRows * numCols: ${numRows * numCols}")
- override def toArray: Array[Double] = values
+ /**
+ * Column-major dense matrix.
+ * The entry values are stored in a single array of doubles with columns listed in sequence.
+ * For example, the following matrix
+ * {{{
+ * 1.0 2.0
+ * 3.0 4.0
+ * 5.0 6.0
+ * }}}
+ * is stored as `[1.0, 3.0, 5.0, 2.0, 4.0, 6.0]`.
+ *
+ * @param numRows number of rows
+ * @param numCols number of columns
+ * @param values matrix entries in column major
+ */
+ def this(numRows: Int, numCols: Int, values: Array[Double]) =
+ this(numRows, numCols, values, false)
override def equals(o: Any) = o match {
case m: DenseMatrix =>
@@ -122,13 +152,22 @@ class DenseMatrix(val numRows: Int, val numCols: Int, val values: Array[Double])
case _ => false
}
- private[mllib] def toBreeze: BM[Double] = new BDM[Double](numRows, numCols, values)
+ private[mllib] def toBreeze: BM[Double] = {
+ if (!isTransposed) {
+ new BDM[Double](numRows, numCols, values)
+ } else {
+ val breezeMatrix = new BDM[Double](numCols, numRows, values)
+ breezeMatrix.t
+ }
+ }
private[mllib] def apply(i: Int): Double = values(i)
private[mllib] def apply(i: Int, j: Int): Double = values(index(i, j))
- private[mllib] def index(i: Int, j: Int): Int = i + numRows * j
+ private[mllib] def index(i: Int, j: Int): Int = {
+ if (!isTransposed) i + numRows * j else j + numCols * i
+ }
private[mllib] def update(i: Int, j: Int, v: Double): Unit = {
values(index(i, j)) = v
@@ -148,7 +187,38 @@ class DenseMatrix(val numRows: Int, val numCols: Int, val values: Array[Double])
this
}
- /** Generate a `SparseMatrix` from the given `DenseMatrix`. */
+ override def transpose: Matrix = new DenseMatrix(numCols, numRows, values, !isTransposed)
+
+ private[spark] override def foreachActive(f: (Int, Int, Double) => Unit): Unit = {
+ if (!isTransposed) {
+ // outer loop over columns
+ var j = 0
+ while (j < numCols) {
+ var i = 0
+ val indStart = j * numRows
+ while (i < numRows) {
+ f(i, j, values(indStart + i))
+ i += 1
+ }
+ j += 1
+ }
+ } else {
+ // outer loop over rows
+ var i = 0
+ while (i < numRows) {
+ var j = 0
+ val indStart = i * numCols
+ while (j < numCols) {
+ f(i, j, values(indStart + j))
+ j += 1
+ }
+ i += 1
+ }
+ }
+ }
+
+ /** Generate a `SparseMatrix` from the given `DenseMatrix`. The new matrix will have isTransposed
+ * set to false. */
def toSparse(): SparseMatrix = {
val spVals: MArrayBuilder[Double] = new MArrayBuilder.ofDouble
val colPtrs: Array[Int] = new Array[Int](numCols + 1)
@@ -157,9 +227,8 @@ class DenseMatrix(val numRows: Int, val numCols: Int, val values: Array[Double])
var j = 0
while (j < numCols) {
var i = 0
- val indStart = j * numRows
while (i < numRows) {
- val v = values(indStart + i)
+ val v = values(index(i, j))
if (v != 0.0) {
rowIndices += i
spVals += v
@@ -271,49 +340,73 @@ object DenseMatrix {
* @param rowIndices the row index of the entry. They must be in strictly increasing order for each
* column
* @param values non-zero matrix entries in column major
+ * @param isTransposed whether the matrix is transposed. If true, the matrix can be considered
+ * Compressed Sparse Row (CSR) format, where `colPtrs` behaves as rowPtrs,
+ * and `rowIndices` behave as colIndices, and `values` are stored in row major.
*/
class SparseMatrix(
val numRows: Int,
val numCols: Int,
val colPtrs: Array[Int],
val rowIndices: Array[Int],
- val values: Array[Double]) extends Matrix {
+ val values: Array[Double],
+ override val isTransposed: Boolean) extends Matrix {
require(values.length == rowIndices.length, "The number of row indices and values don't match! " +
s"values.length: ${values.length}, rowIndices.length: ${rowIndices.length}")
- require(colPtrs.length == numCols + 1, "The length of the column indices should be the " +
- s"number of columns + 1. Currently, colPointers.length: ${colPtrs.length}, " +
- s"numCols: $numCols")
+ // The Or statement is for the case when the matrix is transposed
+ require(colPtrs.length == numCols + 1 || colPtrs.length == numRows + 1, "The length of the " +
+ "column indices should be the number of columns + 1. Currently, colPointers.length: " +
+ s"${colPtrs.length}, numCols: $numCols")
require(values.length == colPtrs.last, "The last value of colPtrs must equal the number of " +
s"elements. values.length: ${values.length}, colPtrs.last: ${colPtrs.last}")
- override def toArray: Array[Double] = {
- val arr = new Array[Double](numRows * numCols)
- var j = 0
- while (j < numCols) {
- var i = colPtrs(j)
- val indEnd = colPtrs(j + 1)
- val offset = j * numRows
- while (i < indEnd) {
- val rowIndex = rowIndices(i)
- arr(offset + rowIndex) = values(i)
- i += 1
- }
- j += 1
- }
- arr
+ /**
+ * Column-major sparse matrix.
+ * The entry values are stored in Compressed Sparse Column (CSC) format.
+ * For example, the following matrix
+ * {{{
+ * 1.0 0.0 4.0
+ * 0.0 3.0 5.0
+ * 2.0 0.0 6.0
+ * }}}
+ * is stored as `values: [1.0, 2.0, 3.0, 4.0, 5.0, 6.0]`,
+ * `rowIndices=[0, 2, 1, 0, 1, 2]`, `colPointers=[0, 2, 3, 6]`.
+ *
+ * @param numRows number of rows
+ * @param numCols number of columns
+ * @param colPtrs the index corresponding to the start of a new column
+ * @param rowIndices the row index of the entry. They must be in strictly increasing
+ * order for each column
+ * @param values non-zero matrix entries in column major
+ */
+ def this(
+ numRows: Int,
+ numCols: Int,
+ colPtrs: Array[Int],
+ rowIndices: Array[Int],
+ values: Array[Double]) = this(numRows, numCols, colPtrs, rowIndices, values, false)
+
+ private[mllib] def toBreeze: BM[Double] = {
+ if (!isTransposed) {
+ new BSM[Double](values, numRows, numCols, colPtrs, rowIndices)
+ } else {
+ val breezeMatrix = new BSM[Double](values, numCols, numRows, colPtrs, rowIndices)
+ breezeMatrix.t
+ }
}
- private[mllib] def toBreeze: BM[Double] =
- new BSM[Double](values, numRows, numCols, colPtrs, rowIndices)
-
private[mllib] def apply(i: Int, j: Int): Double = {
val ind = index(i, j)
if (ind < 0) 0.0 else values(ind)
}
private[mllib] def index(i: Int, j: Int): Int = {
- Arrays.binarySearch(rowIndices, colPtrs(j), colPtrs(j + 1), i)
+ if (!isTransposed) {
+ Arrays.binarySearch(rowIndices, colPtrs(j), colPtrs(j + 1), i)
+ } else {
+ Arrays.binarySearch(rowIndices, colPtrs(i), colPtrs(i + 1), j)
+ }
}
private[mllib] def update(i: Int, j: Int, v: Double): Unit = {
@@ -322,7 +415,7 @@ class SparseMatrix(
throw new NoSuchElementException("The given row and column indices correspond to a zero " +
"value. Only non-zero elements in Sparse Matrices can be updated.")
} else {
- values(index(i, j)) = v
+ values(ind) = v
}
}
@@ -341,7 +434,38 @@ class SparseMatrix(
this
}
- /** Generate a `DenseMatrix` from the given `SparseMatrix`. */
+ override def transpose: Matrix =
+ new SparseMatrix(numCols, numRows, colPtrs, rowIndices, values, !isTransposed)
+
+ private[spark] override def foreachActive(f: (Int, Int, Double) => Unit): Unit = {
+ if (!isTransposed) {
+ var j = 0
+ while (j < numCols) {
+ var idx = colPtrs(j)
+ val idxEnd = colPtrs(j + 1)
+ while (idx < idxEnd) {
+ f(rowIndices(idx), j, values(idx))
+ idx += 1
+ }
+ j += 1
+ }
+ } else {
+ var i = 0
+ while (i < numRows) {
+ var idx = colPtrs(i)
+ val idxEnd = colPtrs(i + 1)
+ while (idx < idxEnd) {
+ val j = rowIndices(idx)
+ f(i, j, values(idx))
+ idx += 1
+ }
+ i += 1
+ }
+ }
+ }
+
+ /** Generate a `DenseMatrix` from the given `SparseMatrix`. The new matrix will have isTransposed
+ * set to false. */
def toDense(): DenseMatrix = {
new DenseMatrix(numRows, numCols, toArray)
}
@@ -557,10 +681,9 @@ object Matrices {
private[mllib] def fromBreeze(breeze: BM[Double]): Matrix = {
breeze match {
case dm: BDM[Double] =>
- require(dm.majorStride == dm.rows,
- "Do not support stride size different from the number of rows.")
- new DenseMatrix(dm.rows, dm.cols, dm.data)
+ new DenseMatrix(dm.rows, dm.cols, dm.data, dm.isTranspose)
case sm: BSM[Double] =>
+ // There is no isTranspose flag for sparse matrices in Breeze
new SparseMatrix(sm.rows, sm.cols, sm.colPtrs, sm.rowIndices, sm.data)
case _ =>
throw new UnsupportedOperationException(
@@ -679,46 +802,28 @@ object Matrices {
new DenseMatrix(numRows, numCols, matrices.flatMap(_.toArray))
} else {
var startCol = 0
- val entries: Array[(Int, Int, Double)] = matrices.flatMap {
- case spMat: SparseMatrix =>
- var j = 0
- val colPtrs = spMat.colPtrs
- val rowIndices = spMat.rowIndices
- val values = spMat.values
- val data = new Array[(Int, Int, Double)](values.length)
- val nCols = spMat.numCols
- while (j < nCols) {
- var idx = colPtrs(j)
- while (idx < colPtrs(j + 1)) {
- val i = rowIndices(idx)
- val v = values(idx)
- data(idx) = (i, j + startCol, v)
- idx += 1
+ val entries: Array[(Int, Int, Double)] = matrices.flatMap { mat =>
+ val nCols = mat.numCols
+ mat match {
+ case spMat: SparseMatrix =>
+ val data = new Array[(Int, Int, Double)](spMat.values.length)
+ var cnt = 0
+ spMat.foreachActive { (i, j, v) =>
+ data(cnt) = (i, j + startCol, v)
+ cnt += 1
}
- j += 1
- }
- startCol += nCols
- data
- case dnMat: DenseMatrix =>
- val data = new ArrayBuffer[(Int, Int, Double)]()
- var j = 0
- val nCols = dnMat.numCols
- val nRows = dnMat.numRows
- val values = dnMat.values
- while (j < nCols) {
- var i = 0
- val indStart = j * nRows
- while (i < nRows) {
- val v = values(indStart + i)
+ startCol += nCols
+ data
+ case dnMat: DenseMatrix =>
+ val data = new ArrayBuffer[(Int, Int, Double)]()
+ dnMat.foreachActive { (i, j, v) =>
if (v != 0.0) {
data.append((i, j + startCol, v))
}
- i += 1
}
- j += 1
- }
- startCol += nCols
- data
+ startCol += nCols
+ data
+ }
}
SparseMatrix.fromCOO(numRows, numCols, entries)
}
@@ -744,14 +849,12 @@ object Matrices {
require(numCols == mat.numCols, "The number of rows of the matrices in this sequence, " +
"don't match!")
mat match {
- case sparse: SparseMatrix =>
- hasSparse = true
- case dense: DenseMatrix =>
+ case sparse: SparseMatrix => hasSparse = true
+ case dense: DenseMatrix => // empty on purpose
case _ => throw new IllegalArgumentException("Unsupported matrix format. Expected " +
s"SparseMatrix or DenseMatrix. Instead got: ${mat.getClass}")
}
numRows += mat.numRows
-
}
if (!hasSparse) {
val allValues = new Array[Double](numRows * numCols)
@@ -759,61 +862,37 @@ object Matrices {
matrices.foreach { mat =>
var j = 0
val nRows = mat.numRows
- val values = mat.toArray
- while (j < numCols) {
- var i = 0
+ mat.foreachActive { (i, j, v) =>
val indStart = j * numRows + startRow
- val subMatStart = j * nRows
- while (i < nRows) {
- allValues(indStart + i) = values(subMatStart + i)
- i += 1
- }
- j += 1
+ allValues(indStart + i) = v
}
startRow += nRows
}
new DenseMatrix(numRows, numCols, allValues)
} else {
var startRow = 0
- val entries: Array[(Int, Int, Double)] = matrices.flatMap {
- case spMat: SparseMatrix =>
- var j = 0
- val colPtrs = spMat.colPtrs
- val rowIndices = spMat.rowIndices
- val values = spMat.values
- val data = new Array[(Int, Int, Double)](values.length)
- while (j < numCols) {
- var idx = colPtrs(j)
- while (idx < colPtrs(j + 1)) {
- val i = rowIndices(idx)
- val v = values(idx)
- data(idx) = (i + startRow, j, v)
- idx += 1
+ val entries: Array[(Int, Int, Double)] = matrices.flatMap { mat =>
+ val nRows = mat.numRows
+ mat match {
+ case spMat: SparseMatrix =>
+ val data = new Array[(Int, Int, Double)](spMat.values.length)
+ var cnt = 0
+ spMat.foreachActive { (i, j, v) =>
+ data(cnt) = (i + startRow, j, v)
+ cnt += 1
}
- j += 1
- }
- startRow += spMat.numRows
- data
- case dnMat: DenseMatrix =>
- val data = new ArrayBuffer[(Int, Int, Double)]()
- var j = 0
- val nCols = dnMat.numCols
- val nRows = dnMat.numRows
- val values = dnMat.values
- while (j < nCols) {
- var i = 0
- val indStart = j * nRows
- while (i < nRows) {
- val v = values(indStart + i)
+ startRow += nRows
+ data
+ case dnMat: DenseMatrix =>
+ val data = new ArrayBuffer[(Int, Int, Double)]()
+ dnMat.foreachActive { (i, j, v) =>
if (v != 0.0) {
data.append((i + startRow, j, v))
}
- i += 1
}
- j += 1
- }
- startRow += nRows
- data
+ startRow += nRows
+ data
+ }
}
SparseMatrix.fromCOO(numRows, numCols, entries)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/91426748/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
index 771878e..b0b78ac 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BLASSuite.scala
@@ -169,16 +169,17 @@ class BLASSuite extends FunSuite {
}
test("gemm") {
-
val dA =
new DenseMatrix(4, 3, Array(0.0, 1.0, 0.0, 0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 3.0))
val sA = new SparseMatrix(4, 3, Array(0, 1, 3, 4), Array(1, 0, 2, 3), Array(1.0, 2.0, 1.0, 3.0))
val B = new DenseMatrix(3, 2, Array(1.0, 0.0, 0.0, 0.0, 2.0, 1.0))
val expected = new DenseMatrix(4, 2, Array(0.0, 1.0, 0.0, 0.0, 4.0, 0.0, 2.0, 3.0))
+ val BTman = new DenseMatrix(2, 3, Array(1.0, 0.0, 0.0, 2.0, 0.0, 1.0))
+ val BT = B.transpose
- assert(dA multiply B ~== expected absTol 1e-15)
- assert(sA multiply B ~== expected absTol 1e-15)
+ assert(dA.multiply(B) ~== expected absTol 1e-15)
+ assert(sA.multiply(B) ~== expected absTol 1e-15)
val C1 = new DenseMatrix(4, 2, Array(1.0, 0.0, 2.0, 1.0, 0.0, 0.0, 1.0, 0.0))
val C2 = C1.copy
@@ -188,6 +189,10 @@ class BLASSuite extends FunSuite {
val C6 = C1.copy
val C7 = C1.copy
val C8 = C1.copy
+ val C9 = C1.copy
+ val C10 = C1.copy
+ val C11 = C1.copy
+ val C12 = C1.copy
val expected2 = new DenseMatrix(4, 2, Array(2.0, 1.0, 4.0, 2.0, 4.0, 0.0, 4.0, 3.0))
val expected3 = new DenseMatrix(4, 2, Array(2.0, 2.0, 4.0, 2.0, 8.0, 0.0, 6.0, 6.0))
@@ -202,26 +207,40 @@ class BLASSuite extends FunSuite {
withClue("columns of A don't match the rows of B") {
intercept[Exception] {
- gemm(true, false, 1.0, dA, B, 2.0, C1)
+ gemm(1.0, dA.transpose, B, 2.0, C1)
}
}
- val dAT =
+ val dATman =
new DenseMatrix(3, 4, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0))
- val sAT =
+ val sATman =
new SparseMatrix(3, 4, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0))
- assert(dAT transposeMultiply B ~== expected absTol 1e-15)
- assert(sAT transposeMultiply B ~== expected absTol 1e-15)
-
- gemm(true, false, 1.0, dAT, B, 2.0, C5)
- gemm(true, false, 1.0, sAT, B, 2.0, C6)
- gemm(true, false, 2.0, dAT, B, 2.0, C7)
- gemm(true, false, 2.0, sAT, B, 2.0, C8)
+ val dATT = dATman.transpose
+ val sATT = sATman.transpose
+ val BTT = BTman.transpose.asInstanceOf[DenseMatrix]
+
+ assert(dATT.multiply(B) ~== expected absTol 1e-15)
+ assert(sATT.multiply(B) ~== expected absTol 1e-15)
+ assert(dATT.multiply(BTT) ~== expected absTol 1e-15)
+ assert(sATT.multiply(BTT) ~== expected absTol 1e-15)
+
+ gemm(1.0, dATT, BTT, 2.0, C5)
+ gemm(1.0, sATT, BTT, 2.0, C6)
+ gemm(2.0, dATT, BTT, 2.0, C7)
+ gemm(2.0, sATT, BTT, 2.0, C8)
+ gemm(1.0, dA, BTT, 2.0, C9)
+ gemm(1.0, sA, BTT, 2.0, C10)
+ gemm(2.0, dA, BTT, 2.0, C11)
+ gemm(2.0, sA, BTT, 2.0, C12)
assert(C5 ~== expected2 absTol 1e-15)
assert(C6 ~== expected2 absTol 1e-15)
assert(C7 ~== expected3 absTol 1e-15)
assert(C8 ~== expected3 absTol 1e-15)
+ assert(C9 ~== expected2 absTol 1e-15)
+ assert(C10 ~== expected2 absTol 1e-15)
+ assert(C11 ~== expected3 absTol 1e-15)
+ assert(C12 ~== expected3 absTol 1e-15)
}
test("gemv") {
@@ -233,17 +252,13 @@ class BLASSuite extends FunSuite {
val x = new DenseVector(Array(1.0, 2.0, 3.0))
val expected = new DenseVector(Array(4.0, 1.0, 2.0, 9.0))
- assert(dA multiply x ~== expected absTol 1e-15)
- assert(sA multiply x ~== expected absTol 1e-15)
+ assert(dA.multiply(x) ~== expected absTol 1e-15)
+ assert(sA.multiply(x) ~== expected absTol 1e-15)
val y1 = new DenseVector(Array(1.0, 3.0, 1.0, 0.0))
val y2 = y1.copy
val y3 = y1.copy
val y4 = y1.copy
- val y5 = y1.copy
- val y6 = y1.copy
- val y7 = y1.copy
- val y8 = y1.copy
val expected2 = new DenseVector(Array(6.0, 7.0, 4.0, 9.0))
val expected3 = new DenseVector(Array(10.0, 8.0, 6.0, 18.0))
@@ -257,25 +272,18 @@ class BLASSuite extends FunSuite {
assert(y4 ~== expected3 absTol 1e-15)
withClue("columns of A don't match the rows of B") {
intercept[Exception] {
- gemv(true, 1.0, dA, x, 2.0, y1)
+ gemv(1.0, dA.transpose, x, 2.0, y1)
}
}
-
val dAT =
new DenseMatrix(3, 4, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0))
val sAT =
new SparseMatrix(3, 4, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0))
- assert(dAT transposeMultiply x ~== expected absTol 1e-15)
- assert(sAT transposeMultiply x ~== expected absTol 1e-15)
-
- gemv(true, 1.0, dAT, x, 2.0, y5)
- gemv(true, 1.0, sAT, x, 2.0, y6)
- gemv(true, 2.0, dAT, x, 2.0, y7)
- gemv(true, 2.0, sAT, x, 2.0, y8)
- assert(y5 ~== expected2 absTol 1e-15)
- assert(y6 ~== expected2 absTol 1e-15)
- assert(y7 ~== expected3 absTol 1e-15)
- assert(y8 ~== expected3 absTol 1e-15)
+ val dATT = dAT.transpose
+ val sATT = sAT.transpose
+
+ assert(dATT.multiply(x) ~== expected absTol 1e-15)
+ assert(sATT.multiply(x) ~== expected absTol 1e-15)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/91426748/mllib/src/test/scala/org/apache/spark/mllib/linalg/BreezeMatrixConversionSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BreezeMatrixConversionSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BreezeMatrixConversionSuite.scala
index 73a6d3a..2031032 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/BreezeMatrixConversionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/BreezeMatrixConversionSuite.scala
@@ -36,6 +36,11 @@ class BreezeMatrixConversionSuite extends FunSuite {
assert(mat.numRows === breeze.rows)
assert(mat.numCols === breeze.cols)
assert(mat.values.eq(breeze.data), "should not copy data")
+ // transposed matrix
+ val matTransposed = Matrices.fromBreeze(breeze.t).asInstanceOf[DenseMatrix]
+ assert(matTransposed.numRows === breeze.cols)
+ assert(matTransposed.numCols === breeze.rows)
+ assert(matTransposed.values.eq(breeze.data), "should not copy data")
}
test("sparse matrix to breeze") {
@@ -58,5 +63,9 @@ class BreezeMatrixConversionSuite extends FunSuite {
assert(mat.numRows === breeze.rows)
assert(mat.numCols === breeze.cols)
assert(mat.values.eq(breeze.data), "should not copy data")
+ val matTransposed = Matrices.fromBreeze(breeze.t).asInstanceOf[SparseMatrix]
+ assert(matTransposed.numRows === breeze.cols)
+ assert(matTransposed.numCols === breeze.rows)
+ assert(!matTransposed.values.eq(breeze.data), "has to copy data")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/91426748/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
index a35d0fe..b1ebfde 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/linalg/MatricesSuite.scala
@@ -22,6 +22,9 @@ import java.util.Random
import org.mockito.Mockito.when
import org.scalatest.FunSuite
import org.scalatest.mock.MockitoSugar._
+import scala.collection.mutable.{Map => MutableMap}
+
+import org.apache.spark.mllib.util.TestingUtils._
class MatricesSuite extends FunSuite {
test("dense matrix construction") {
@@ -32,7 +35,6 @@ class MatricesSuite extends FunSuite {
assert(mat.numRows === m)
assert(mat.numCols === n)
assert(mat.values.eq(values), "should not copy data")
- assert(mat.toArray.eq(values), "toArray should not copy data")
}
test("dense matrix construction with wrong dimension") {
@@ -161,6 +163,66 @@ class MatricesSuite extends FunSuite {
assert(deMat1.toArray === deMat2.toArray)
}
+ test("transpose") {
+ val dA =
+ new DenseMatrix(4, 3, Array(0.0, 1.0, 0.0, 0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 3.0))
+ val sA = new SparseMatrix(4, 3, Array(0, 1, 3, 4), Array(1, 0, 2, 3), Array(1.0, 2.0, 1.0, 3.0))
+
+ val dAT = dA.transpose.asInstanceOf[DenseMatrix]
+ val sAT = sA.transpose.asInstanceOf[SparseMatrix]
+ val dATexpected =
+ new DenseMatrix(3, 4, Array(0.0, 2.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 3.0))
+ val sATexpected =
+ new SparseMatrix(3, 4, Array(0, 1, 2, 3, 4), Array(1, 0, 1, 2), Array(2.0, 1.0, 1.0, 3.0))
+
+ assert(dAT.toBreeze === dATexpected.toBreeze)
+ assert(sAT.toBreeze === sATexpected.toBreeze)
+ assert(dA(1, 0) === dAT(0, 1))
+ assert(dA(2, 1) === dAT(1, 2))
+ assert(sA(1, 0) === sAT(0, 1))
+ assert(sA(2, 1) === sAT(1, 2))
+
+ assert(!dA.toArray.eq(dAT.toArray), "has to have a new array")
+ assert(dA.values.eq(dAT.transpose.asInstanceOf[DenseMatrix].values), "should not copy array")
+
+ assert(dAT.toSparse().toBreeze === sATexpected.toBreeze)
+ assert(sAT.toDense().toBreeze === dATexpected.toBreeze)
+ }
+
+ test("foreachActive") {
+ val m = 3
+ val n = 2
+ val values = Array(1.0, 2.0, 4.0, 5.0)
+ val allValues = Array(1.0, 2.0, 0.0, 0.0, 4.0, 5.0)
+ val colPtrs = Array(0, 2, 4)
+ val rowIndices = Array(0, 1, 1, 2)
+
+ val sp = new SparseMatrix(m, n, colPtrs, rowIndices, values)
+ val dn = new DenseMatrix(m, n, allValues)
+
+ val dnMap = MutableMap[(Int, Int), Double]()
+ dn.foreachActive { (i, j, value) =>
+ dnMap.put((i, j), value)
+ }
+ assert(dnMap.size === 6)
+ assert(dnMap(0, 0) === 1.0)
+ assert(dnMap(1, 0) === 2.0)
+ assert(dnMap(2, 0) === 0.0)
+ assert(dnMap(0, 1) === 0.0)
+ assert(dnMap(1, 1) === 4.0)
+ assert(dnMap(2, 1) === 5.0)
+
+ val spMap = MutableMap[(Int, Int), Double]()
+ sp.foreachActive { (i, j, value) =>
+ spMap.put((i, j), value)
+ }
+ assert(spMap.size === 4)
+ assert(spMap(0, 0) === 1.0)
+ assert(spMap(1, 0) === 2.0)
+ assert(spMap(1, 1) === 4.0)
+ assert(spMap(2, 1) === 5.0)
+ }
+
test("horzcat, vertcat, eye, speye") {
val m = 3
val n = 2
@@ -168,9 +230,20 @@ class MatricesSuite extends FunSuite {
val allValues = Array(1.0, 2.0, 0.0, 0.0, 4.0, 5.0)
val colPtrs = Array(0, 2, 4)
val rowIndices = Array(0, 1, 1, 2)
+ // transposed versions
+ val allValuesT = Array(1.0, 0.0, 2.0, 4.0, 0.0, 5.0)
+ val colPtrsT = Array(0, 1, 3, 4)
+ val rowIndicesT = Array(0, 0, 1, 1)
val spMat1 = new SparseMatrix(m, n, colPtrs, rowIndices, values)
val deMat1 = new DenseMatrix(m, n, allValues)
+ val spMat1T = new SparseMatrix(n, m, colPtrsT, rowIndicesT, values)
+ val deMat1T = new DenseMatrix(n, m, allValuesT)
+
+ // should equal spMat1 & deMat1 respectively
+ val spMat1TT = spMat1T.transpose
+ val deMat1TT = deMat1T.transpose
+
val deMat2 = Matrices.eye(3)
val spMat2 = Matrices.speye(3)
val deMat3 = Matrices.eye(2)
@@ -180,7 +253,6 @@ class MatricesSuite extends FunSuite {
val spHorz2 = Matrices.horzcat(Array(spMat1, deMat2))
val spHorz3 = Matrices.horzcat(Array(deMat1, spMat2))
val deHorz1 = Matrices.horzcat(Array(deMat1, deMat2))
-
val deHorz2 = Matrices.horzcat(Array[Matrix]())
assert(deHorz1.numRows === 3)
@@ -195,8 +267,8 @@ class MatricesSuite extends FunSuite {
assert(deHorz2.numCols === 0)
assert(deHorz2.toArray.length === 0)
- assert(deHorz1.toBreeze.toDenseMatrix === spHorz2.toBreeze.toDenseMatrix)
- assert(spHorz2.toBreeze === spHorz3.toBreeze)
+ assert(deHorz1 ~== spHorz2.asInstanceOf[SparseMatrix].toDense absTol 1e-15)
+ assert(spHorz2 ~== spHorz3 absTol 1e-15)
assert(spHorz(0, 0) === 1.0)
assert(spHorz(2, 1) === 5.0)
assert(spHorz(0, 2) === 1.0)
@@ -212,6 +284,17 @@ class MatricesSuite extends FunSuite {
assert(deHorz1(2, 4) === 1.0)
assert(deHorz1(1, 4) === 0.0)
+ // containing transposed matrices
+ val spHorzT = Matrices.horzcat(Array(spMat1TT, spMat2))
+ val spHorz2T = Matrices.horzcat(Array(spMat1TT, deMat2))
+ val spHorz3T = Matrices.horzcat(Array(deMat1TT, spMat2))
+ val deHorz1T = Matrices.horzcat(Array(deMat1TT, deMat2))
+
+ assert(deHorz1T ~== deHorz1 absTol 1e-15)
+ assert(spHorzT ~== spHorz absTol 1e-15)
+ assert(spHorz2T ~== spHorz2 absTol 1e-15)
+ assert(spHorz3T ~== spHorz3 absTol 1e-15)
+
intercept[IllegalArgumentException] {
Matrices.horzcat(Array(spMat1, spMat3))
}
@@ -238,8 +321,8 @@ class MatricesSuite extends FunSuite {
assert(deVert2.numCols === 0)
assert(deVert2.toArray.length === 0)
- assert(deVert1.toBreeze.toDenseMatrix === spVert2.toBreeze.toDenseMatrix)
- assert(spVert2.toBreeze === spVert3.toBreeze)
+ assert(deVert1 ~== spVert2.asInstanceOf[SparseMatrix].toDense absTol 1e-15)
+ assert(spVert2 ~== spVert3 absTol 1e-15)
assert(spVert(0, 0) === 1.0)
assert(spVert(2, 1) === 5.0)
assert(spVert(3, 0) === 1.0)
@@ -251,6 +334,17 @@ class MatricesSuite extends FunSuite {
assert(deVert1(3, 1) === 0.0)
assert(deVert1(4, 1) === 1.0)
+ // containing transposed matrices
+ val spVertT = Matrices.vertcat(Array(spMat1TT, spMat3))
+ val deVert1T = Matrices.vertcat(Array(deMat1TT, deMat3))
+ val spVert2T = Matrices.vertcat(Array(spMat1TT, deMat3))
+ val spVert3T = Matrices.vertcat(Array(deMat1TT, spMat3))
+
+ assert(deVert1T ~== deVert1 absTol 1e-15)
+ assert(spVertT ~== spVert absTol 1e-15)
+ assert(spVert2T ~== spVert2 absTol 1e-15)
+ assert(spVert3T ~== spVert3 absTol 1e-15)
+
intercept[IllegalArgumentException] {
Matrices.vertcat(Array(spMat1, spMat2))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/91426748/project/MimaExcludes.scala
----------------------------------------------------------------------
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index bc5d81f..af0b0eb 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -53,6 +53,20 @@ object MimaExcludes {
ProblemFilters.exclude[MissingMethodProblem](
"org.apache.spark.mllib.linalg.Matrices.rand")
) ++ Seq(
+ // SPARK-5321
+ ProblemFilters.exclude[MissingMethodProblem](
+ "org.apache.spark.mllib.linalg.SparseMatrix.transposeMultiply"),
+ ProblemFilters.exclude[MissingMethodProblem](
+ "org.apache.spark.mllib.linalg.Matrix.transpose"),
+ ProblemFilters.exclude[MissingMethodProblem](
+ "org.apache.spark.mllib.linalg.DenseMatrix.transposeMultiply"),
+ ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.mllib.linalg.Matrix." +
+ "org$apache$spark$mllib$linalg$Matrix$_setter_$isTransposed_="),
+ ProblemFilters.exclude[MissingMethodProblem](
+ "org.apache.spark.mllib.linalg.Matrix.isTransposed"),
+ ProblemFilters.exclude[MissingMethodProblem](
+ "org.apache.spark.mllib.linalg.Matrix.foreachActive")
+ ) ++ Seq(
// SPARK-3325
ProblemFilters.exclude[MissingMethodProblem](
"org.apache.spark.streaming.api.java.JavaDStreamLike.print"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org