You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by ap...@apache.org on 2017/01/26 04:24:19 UTC
[1/5] mahout git commit: MAHOUT-1885: Inital commit of VCL bindings.
closes apache/mahout#269 closes apache/mahout#261
Repository: mahout
Updated Branches:
refs/heads/master 84e90ed23 -> 034790cce
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/SrMatDnMatProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/SrMatDnMatProdExpression.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/SrMatDnMatProdExpression.scala
new file mode 100644
index 0000000..24d2c7b
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/SrMatDnMatProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::compressed_matrix<double>, " +
+ "const viennacl::matrix_base<double>, " +
+ "viennacl::op_prod>"))
+class SrMatDnMatProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VCLVector.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VCLVector.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VCLVector.scala
new file mode 100644
index 0000000..f0e3010
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VCLVector.scala
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp._
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::vector<double>"))
+final class VCLVector(defaultCtr: Boolean = true) extends VectorBase {
+
+ if (defaultCtr) allocate()
+
+ def this(){
+ this(false)
+ allocate()
+ }
+
+ def this(i: Int){
+ this(false)
+ allocate(i)
+ }
+
+ def this(size: Int, ctx: Context = new Context(Context.MAIN_MEMORY)) {
+ this(false)
+ allocate(size, ctx)
+ }
+
+ def this(@Const @ByRef ve: VecMultExpression) {
+ this(false)
+ allocate(ve)
+ }
+
+ def this(@Const @ByRef vmp: MatVecProdExpression) {
+ this(false)
+ allocate(vmp)
+ }
+
+// conflicting with the next signature as MemHandle is a pointer and so is a DoublePointer..
+// leave out for now.
+//
+// def this(h: MemHandle , vec_size: Int, vec_start: Int = 0, vec_stride: Int = 1) {
+// this(false)
+// allocate(h, vec_size, vec_start, vec_stride)
+// }
+
+ def this(ptr_to_mem: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))mem_type : Int,
+ vec_size: Int,
+ start: Int = 0,
+ stride: Int = 1) {
+
+ this(false)
+ allocate(ptr_to_mem, mem_type, vec_size, start, stride)
+ ptrs += ptr_to_mem
+ }
+
+ def this(@Const @ByRef vc: VCLVector) {
+ this(false)
+ allocate(vc)
+ }
+ def this(@Const @ByRef vb: VectorBase) {
+ this(false)
+ allocate(vb)
+ }
+
+ @native protected def allocate()
+
+ @native protected def allocate(size: Int)
+
+ @native protected def allocate(size: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(@Const @ByRef ve: VecMultExpression)
+
+ @native protected def allocate(@Const @ByRef ve: MatVecProdExpression)
+
+ @native protected def allocate(@Const @ByRef vb: VCLVector)
+
+ @native protected def allocate(@Const @ByRef vb: VectorBase)
+
+
+// @native protected def allocate(h: MemHandle , vec_size: Int,
+// vec_start: Int,
+// vec_stride: Int)
+
+ @native protected def allocate(ptr_to_mem: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))mem_type : Int,
+ vec_size: Int,
+ start: Int,
+ stride: Int)
+
+ @Name(Array("viennacl::vector<double>::self_type"))
+ def selfType:VectorBase = this.asInstanceOf[VectorBase]
+
+
+ @native def switch_memory_context(@ByVal context: Context): Unit
+
+// Swaps the handles of two vectors by swapping the OpenCL handles only, no data copy.
+// @native def fast_swap(@ByVal other: VCLVector): VectorBase
+
+// add this operator in for tests many more can be added
+// @Name(Array("operator*"))
+// @native @ByPtr def *(i: Int): VectorMultExpression
+
+
+
+}
+
+object VCLVector {
+ Context.loadLib()
+}
+
+
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VecMultExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VecMultExpression.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VecMultExpression.scala
new file mode 100644
index 0000000..1904151
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VecMultExpression.scala
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("vector_expression<const viennacl::vector_base<double>," +
+ "const double, viennacl::op_mult >"))
+class VecMultExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VectorBase.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VectorBase.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VectorBase.scala
new file mode 100644
index 0000000..43ae39d
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/VectorBase.scala
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import java.nio._
+
+import org.bytedeco.javacpp._
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::vector_base<double>"))
+class VectorBase extends Pointer {
+
+ protected val ptrs = new ArrayBuffer[Pointer]()
+
+ override def deallocate(deallocate: Boolean): Unit = {
+ super.deallocate(deallocate)
+ ptrs.foreach(_.close())
+ }
+
+ // size of the vec elements
+ @native @Const def size(): Int
+
+ // size of the vec elements + padding
+ @native @Const def internal_size(): Int
+
+ // handle to the vec element buffer
+ @native @Const @ByRef def handle: MemHandle
+
+// // add this operator in for tests many more can be added
+// @Name(Array("operator* "))
+// @native def *(i: Int): VectorMultExpression
+
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/package.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/package.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/package.scala
new file mode 100644
index 0000000..8c3743a
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/package.scala
@@ -0,0 +1,434 @@
+package org.apache.mahout.viennacl
+
+import java.nio._
+
+import org.apache.mahout.math._
+import scalabindings._
+import RLikeOps._
+import org.apache.mahout.math.backend.incore._
+import scala.collection.JavaConversions._
+import org.apache.mahout.viennacl.opencl.javacpp.{CompressedMatrix, Context, DenseRowMatrix, Functions, VCLVector}
+import org.apache.mahout.viennacl.opencl.javacpp.Context
+import org.bytedeco.javacpp.{DoublePointer, IntPointer}
+
+
+
+package object opencl {
+
+ type IntConvertor = Int => Int
+
+ def toVclDenseRM(src: Matrix, vclCtx: Context = new Context(Context.MAIN_MEMORY)): DenseRowMatrix = {
+ vclCtx.memoryType match {
+ case Context.MAIN_MEMORY \u21d2
+ val vclMx = new DenseRowMatrix(
+ data = repackRowMajor(src, src.nrow, src.ncol),
+ nrow = src.nrow,
+ ncol = src.ncol,
+ ctx = vclCtx
+ )
+ vclMx
+ case _ \u21d2
+ val vclMx = new DenseRowMatrix(src.nrow, src.ncol, vclCtx)
+ fastCopy(src, vclMx)
+ vclMx
+ }
+ }
+
+
+ /**
+ * Convert a dense row VCL matrix to mahout matrix.
+ *
+ * @param src
+ * @return
+ */
+ def fromVclDenseRM(src: DenseRowMatrix): Matrix = {
+ val nrowIntern = src.internalnrow
+ val ncolIntern = src.internalncol
+
+ // A technical debt here:
+
+ // We do double copying here, this is obviously suboptimal, but hopefully we'll compensate
+ // this with gains from running superlinear algorithms in VCL.
+ val dbuff = new DoublePointer(nrowIntern * ncolIntern)
+ Functions.fastCopy(src, dbuff)
+ var srcOffset = 0
+ val ncol = src.ncol
+ val rows = for (irow \u2190 0 until src.nrow) yield {
+
+ val rowvec = new Array[Double](ncol)
+ dbuff.position(srcOffset).get(rowvec)
+
+ srcOffset += ncolIntern
+ rowvec
+ }
+
+ // Always! use shallow = true to avoid yet another copying.
+ new DenseMatrix(rows.toArray, true)
+ }
+
+ def fastCopy(mxSrc: Matrix, dst: DenseRowMatrix) = {
+ val nrowIntern = dst.internalnrow
+ val ncolIntern = dst.internalncol
+
+ assert(nrowIntern >= mxSrc.nrow && ncolIntern >= mxSrc.ncol)
+
+ val rmajorData = repackRowMajor(mxSrc, nrowIntern, ncolIntern)
+ Functions.fastCopy(rmajorData, new DoublePointer(rmajorData).position(rmajorData.limit()), dst)
+
+ rmajorData.close()
+ }
+
+ private def repackRowMajor(mx: Matrix, nrowIntern: Int, ncolIntern: Int): DoublePointer = {
+
+ assert(mx.nrow <= nrowIntern && mx.ncol <= ncolIntern)
+
+ val dbuff = new DoublePointer(nrowIntern * ncolIntern)
+
+ mx match {
+ case dm: DenseMatrix \u21d2
+ val valuesF = classOf[DenseMatrix].getDeclaredField("values")
+ valuesF.setAccessible(true)
+ val values = valuesF.get(dm).asInstanceOf[Array[Array[Double]]]
+ var dstOffset = 0
+ for (irow \u2190 0 until mx.nrow) {
+ val rowarr = values(irow)
+ dbuff.position(dstOffset).put(rowarr, 0, rowarr.size min ncolIntern)
+ dstOffset += ncolIntern
+ }
+ dbuff.position(0)
+ case _ \u21d2
+ // Naive copying. Could be sped up for a DenseMatrix. TODO.
+ for (row \u2190 mx) {
+ val dstOffset = row.index * ncolIntern
+ for (el \u2190 row.nonZeroes) dbuff.put(dstOffset + el.index, el)
+ }
+ }
+
+ dbuff
+ }
+
+ /**
+ *
+ * @param mxSrc
+ * @param ctx
+ * @return
+ */
+ def toVclCmpMatrixAlt(mxSrc: Matrix, ctx: Context): CompressedMatrix = {
+
+ // use repackCSR(matrix, ctx) to convert all ints to unsigned ints if Context is Ocl
+ // val (jumpers, colIdcs, els) = repackCSRAlt(mxSrc)
+ val (jumpers, colIdcs, els) = repackCSR(mxSrc, ctx)
+
+ val compMx = new CompressedMatrix(mxSrc.nrow, mxSrc.ncol, els.capacity().toInt, ctx)
+ compMx.set(jumpers, colIdcs, els, mxSrc.nrow, mxSrc.ncol, els.capacity().toInt)
+ compMx
+ }
+
+ private def repackCSRAlt(mx: Matrix): (IntPointer, IntPointer, DoublePointer) = {
+ val nzCnt = mx.map(_.getNumNonZeroElements).sum
+ val jumpers = new IntPointer(mx.nrow + 1L)
+ val colIdcs = new IntPointer(nzCnt + 0L)
+ val els = new DoublePointer(nzCnt)
+ var posIdx = 0
+
+ var sortCols = false
+
+ // Row-wise loop. Rows may not necessarily come in order. But we have to have them in-order.
+ for (irow \u2190 0 until mx.nrow) {
+
+ val row = mx(irow, ::)
+ jumpers.put(irow.toLong, posIdx)
+
+ // Remember row start index in case we need to restart conversion of this row if out-of-order
+ // column index is detected
+ val posIdxStart = posIdx
+
+ // Retry loop: normally we are done in one pass thru it unless we need to re-run it because
+ // out-of-order column was detected.
+ var done = false
+ while (!done) {
+
+ // Is the sorting mode on?
+ if (sortCols) {
+
+ // Sorting of column indices is on. So do it.
+ row.nonZeroes()
+ // Need to convert to a strict collection out of iterator
+ .map(el \u21d2 el.index \u2192 el.get)
+ // Sorting requires Sequence api
+ .toSeq
+ // Sort by column index
+ .sortBy(_._1)
+ // Flush to the CSR buffers.
+ .foreach { case (index, v) \u21d2
+ colIdcs.put(posIdx.toLong, index)
+ els.put(posIdx.toLong, v)
+ posIdx += 1
+ }
+
+ // Never need to retry if we are already in the sorting mode.
+ done = true
+
+ } else {
+
+ // Try to run unsorted conversion here, switch lazily to sorted if out-of-order column is
+ // detected.
+ var lastCol = 0
+ val nzIter = row.nonZeroes().iterator()
+ var abortNonSorted = false
+
+ while (nzIter.hasNext && !abortNonSorted) {
+
+ val el = nzIter.next()
+ val index = el.index
+
+ if (index < lastCol) {
+
+ // Out of order detected: abort inner loop, reset posIdx and retry with sorting on.
+ abortNonSorted = true
+ sortCols = true
+ posIdx = posIdxStart
+
+ } else {
+
+ // Still in-order: save element and column, continue.
+ els.put(posIdx, el)
+ colIdcs.put(posIdx.toLong, index)
+ posIdx += 1
+
+ // Remember last column seen.
+ lastCol = index
+ }
+ } // inner non-sorted
+
+ // Do we need to re-run this row with sorting?
+ done = !abortNonSorted
+
+ } // if (sortCols)
+
+ } // while (!done) retry loop
+
+ } // row-wise loop
+
+ // Make sure Mahout matrix did not cheat on non-zero estimate.
+ assert(posIdx == nzCnt)
+
+ jumpers.put(mx.nrow.toLong, nzCnt)
+
+ (jumpers, colIdcs, els)
+ }
+
+ // same as repackCSRAlt except converts to jumpers, colIdcs to unsigned ints before setting
+ private def repackCSR(mx: Matrix, context: Context): (IntPointer, IntPointer, DoublePointer) = {
+ val nzCnt = mx.map(_.getNumNonZeroElements).sum
+ val jumpers = new IntPointer(mx.nrow + 1L)
+ val colIdcs = new IntPointer(nzCnt + 0L)
+ val els = new DoublePointer(nzCnt)
+ var posIdx = 0
+
+ var sortCols = false
+
+ def convertInt: IntConvertor = if(context.memoryType == Context.OPENCL_MEMORY) {
+ int2cl_uint
+ } else {
+ i: Int => i: Int
+ }
+
+ // Row-wise loop. Rows may not necessarily come in order. But we have to have them in-order.
+ for (irow \u2190 0 until mx.nrow) {
+
+ val row = mx(irow, ::)
+ jumpers.put(irow.toLong, posIdx)
+
+ // Remember row start index in case we need to restart conversion of this row if out-of-order
+ // column index is detected
+ val posIdxStart = posIdx
+
+ // Retry loop: normally we are done in one pass thru it unless we need to re-run it because
+ // out-of-order column was detected.
+ var done = false
+ while (!done) {
+
+ // Is the sorting mode on?
+ if (sortCols) {
+
+ // Sorting of column indices is on. So do it.
+ row.nonZeroes()
+ // Need to convert to a strict collection out of iterator
+ .map(el \u21d2 el.index \u2192 el.get)
+ // Sorting requires Sequence api
+ .toIndexedSeq
+ // Sort by column index
+ .sortBy(_._1)
+ // Flush to the CSR buffers.
+ .foreach { case (index, v) \u21d2
+ // convert to cl_uint if context is OCL
+ colIdcs.put(posIdx.toLong, convertInt(index))
+ els.put(posIdx.toLong, v)
+ posIdx += 1
+ }
+
+ // Never need to retry if we are already in the sorting mode.
+ done = true
+
+ } else {
+
+ // Try to run unsorted conversion here, switch lazily to sorted if out-of-order column is
+ // detected.
+ var lastCol = 0
+ val nzIter = row.nonZeroes().iterator()
+ var abortNonSorted = false
+
+ while (nzIter.hasNext && !abortNonSorted) {
+
+ val el = nzIter.next()
+ val index = el.index
+
+ if (index < lastCol) {
+
+ // Out of order detected: abort inner loop, reset posIdx and retry with sorting on.
+ abortNonSorted = true
+ sortCols = true
+ posIdx = posIdxStart
+
+ } else {
+
+ // Still in-order: save element and column, continue.
+ els.put(posIdx, el)
+ // convert to cl_uint if context is OCL
+ colIdcs.put(posIdx.toLong, convertInt(index))
+ posIdx += 1
+
+ // Remember last column seen.
+ lastCol = index
+ }
+ } // inner non-sorted
+
+ // Do we need to re-run this row with sorting?
+ done = !abortNonSorted
+
+ } // if (sortCols)
+
+ } // while (!done) retry loop
+
+ } // row-wise loop
+
+ // Make sure Mahout matrix did not cheat on non-zero estimate.
+ assert(posIdx == nzCnt)
+
+ // convert to cl_uint if context is OCL
+ jumpers.put(mx.nrow.toLong, convertInt(nzCnt))
+
+ (jumpers, colIdcs, els)
+ }
+
+
+
+ def fromVclCompressedMatrix(src: CompressedMatrix): Matrix = {
+ val m = src.size1
+ val n = src.size2
+ val NNz = src.nnz
+
+ val row_ptr_handle = src.handle1
+ val col_idx_handle = src.handle2
+ val element_handle = src.handle
+
+ val row_ptr = new IntPointer((m + 1).toLong)
+ val col_idx = new IntPointer(NNz.toLong)
+ val values = new DoublePointer(NNz.toLong)
+
+ Functions.memoryReadInt(row_ptr_handle, 0, (m + 1) * 4, row_ptr, false)
+ Functions.memoryReadInt(col_idx_handle, 0, NNz * 4, col_idx, false)
+ Functions.memoryReadDouble(element_handle, 0, NNz * 8, values, false)
+
+ val rowPtr = row_ptr.asBuffer()
+ val colIdx = col_idx.asBuffer()
+ val vals = values.asBuffer()
+
+ rowPtr.rewind()
+ colIdx.rewind()
+ vals.rewind()
+
+
+ val srMx = new SparseRowMatrix(m, n)
+
+ // read the values back into the matrix
+ var j = 0
+ // row wise, copy any non-zero elements from row(i-1,::)
+ for (i <- 1 to m) {
+ // for each nonzero element, set column col(idx(j) value to vals(j)
+ while (j < rowPtr.get(i)) {
+ srMx(i - 1, colIdx.get(j)) = vals.get(j)
+ j += 1
+ }
+ }
+ srMx
+ }
+
+ def toVclVec(vec: Vector, ctx: Context): VCLVector = {
+
+ vec match {
+ case vec: DenseVector => {
+ val valuesF = classOf[DenseVector].getDeclaredField("values")
+ valuesF.setAccessible(true)
+ val values = valuesF.get(vec).asInstanceOf[Array[Double]]
+ val el_ptr = new DoublePointer(values.length.toLong)
+ el_ptr.put(values, 0, values.length)
+
+ new VCLVector(el_ptr, ctx.memoryType, values.length)
+ }
+
+ case vec: SequentialAccessSparseVector => {
+ val it = vec.iterateNonZero
+ val size = vec.size()
+ val el_ptr = new DoublePointer(size.toLong)
+ while (it.hasNext) {
+ val el: Vector.Element = it.next
+ el_ptr.put(el.index, el.get())
+ }
+ new VCLVector(el_ptr, ctx.memoryType, size)
+ }
+
+ case vec: RandomAccessSparseVector => {
+ val it = vec.iterateNonZero
+ val size = vec.size()
+ val el_ptr = new DoublePointer(size.toLong)
+ while (it.hasNext) {
+ val el: Vector.Element = it.next
+ el_ptr.put(el.index, el.get())
+ }
+ new VCLVector(el_ptr, ctx.memoryType, size)
+ }
+ case _ => throw new IllegalArgumentException("Vector sub-type not supported.")
+ }
+
+ }
+
+ def fromVClVec(vclVec: VCLVector): Vector = {
+ val size = vclVec.size
+ val element_handle = vclVec.handle
+ val ele_ptr = new DoublePointer(size)
+ Functions.memoryReadDouble(element_handle, 0, size * 8, ele_ptr, false)
+
+ // for now just assume its dense since we only have one flavor of
+ // VCLVector
+ val mVec = new DenseVector(size)
+ for (i <- 0 until size) {
+ mVec.setQuick(i, ele_ptr.get(i + 0L))
+ }
+
+ mVec
+ }
+
+
+ // TODO: Fix this? cl_uint must be an unsigned int per each machine's representation of such.
+ // this is currently not working anyways.
+ // cl_uint is needed for OpenCl sparse Buffers
+ // per https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/scalarDataTypes.html
+ // it is simply an unsigned int, so strip the sign.
+ def int2cl_uint(i: Int): Int = {
+ ((i >>> 1) << 1) + (i & 1)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/test/scala/org/apache/mahout/viennacl/opencl/ViennaCLSuiteVCL.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/test/scala/org/apache/mahout/viennacl/opencl/ViennaCLSuiteVCL.scala b/viennacl/src/test/scala/org/apache/mahout/viennacl/opencl/ViennaCLSuiteVCL.scala
new file mode 100644
index 0000000..c433534
--- /dev/null
+++ b/viennacl/src/test/scala/org/apache/mahout/viennacl/opencl/ViennaCLSuiteVCL.scala
@@ -0,0 +1,427 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.opencl.viennacl
+
+import org.apache.mahout.math._
+import org.apache.mahout.math.scalabindings.RLikeOps._
+import org.apache.mahout.viennacl.opencl.javacpp.CompressedMatrix
+import org.apache.mahout.viennacl.opencl._
+import org.apache.mahout.viennacl.opencl.javacpp.Functions._
+import org.apache.mahout.viennacl.opencl.javacpp.LinalgFunctions._
+import org.apache.mahout.viennacl.opencl.javacpp.{Context, LinalgFunctions, VCLVector, _}
+import org.bytedeco.javacpp.DoublePointer
+import org.scalatest.{FunSuite, Matchers}
+
+import scala.util.Random
+
+class ViennaCLSuiteVCL extends FunSuite with Matchers {
+
+ test("row-major viennacl::matrix") {
+
+ // Just to make sure the javacpp library is loaded:
+ Context.loadLib()
+
+ val m = 20
+ val n = 30
+ val data = new DoublePointer(m * n)
+ val buff = data.asBuffer()
+ // Fill with some noise
+ while (buff.remaining() > 0) buff.put(Random.nextDouble())
+
+ // Create row-major matrix with OpenCL
+ val openClCtx = new Context(Context.OPENCL_MEMORY)
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val oclMx = new DenseRowMatrix(m, n, openClCtx)
+ val cpuMx = new DenseRowMatrix(data = data, nrow = m, ncol = n, hostClCtx)
+
+ oclMx.memoryDomain shouldBe Context.OPENCL_MEMORY
+
+ // Apparently, this doesn't really switch any contexts? any how, uncommenting this causes
+ // subsequent out-of-resources OCL error for me in other tests. Perhaps we shouldnt' really
+ // do cross-memory-domain assigns?
+
+ // oclMx := cpuMx
+
+ // Did it change memory domain? that may explain the OCL resource leak.
+ info(s"OCL matrix memory domain after assgn=${oclMx.memoryDomain}")
+ oclMx.memoryDomain shouldBe Context.OPENCL_MEMORY
+
+
+ // And free.
+ cpuMx.close()
+ oclMx.close()
+
+ }
+
+ test("dense vcl mmul with fast_copy") {
+
+ import LinalgFunctions._
+
+ val vclCtx = new Context(Context.OPENCL_MEMORY)
+
+ val m = 20
+ val n = 30
+ val s = 40
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, s)
+ val mxB = new DenseMatrix(s, n)
+
+ // add some data
+ mxA := { (_, _, _) => r.nextDouble() }
+ mxB := { (_, _, _) => r.nextDouble() }
+
+ // time Mahout MMul
+ // mxC = mxA %*% mxB via Mahout MMul
+ val mxCControl = mxA %*% mxB
+
+ val vclA = toVclDenseRM(mxA, vclCtx)
+ val vclB = toVclDenseRM(mxB, vclCtx)
+
+ val vclC = new DenseRowMatrix(prod(vclA, vclB))
+
+ val mxC = fromVclDenseRM(vclC)
+
+ vclA.close()
+ vclB.close()
+ vclC.close()
+
+ // So did we compute it correctly?
+ (mxC - mxA %*% mxB).norm / m / n should be < 1e-16
+
+ vclCtx.deallocate()
+ vclCtx.close()
+
+ }
+
+ test("mmul microbenchmark") {
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val memCtx = new Context(Context.MAIN_MEMORY)
+
+ val m = 3000
+ val n = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, s)
+ val mxB = new DenseMatrix(s, n)
+
+ // add some data
+ mxA := { (_, _, _) => r.nextDouble() }
+ mxB := { (_, _, _) => r.nextDouble() }
+
+ var ms = System.currentTimeMillis()
+ mxA %*% mxB
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout multiplication time: $ms ms.")
+
+ import LinalgFunctions._
+
+ // openCL time, including copying:
+ {
+ ms = System.currentTimeMillis()
+ val oclA = toVclDenseRM(mxA, oclCtx)
+ val oclB = toVclDenseRM(mxB, oclCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+ }
+
+ // openMP/cpu time, including copying:
+ {
+ ms = System.currentTimeMillis()
+ val ompA = toVclDenseRM(mxA, memCtx)
+ val ompB = toVclDenseRM(mxB, memCtx)
+ val ompC = new DenseRowMatrix(prod(ompA, ompB))
+ val mxC = fromVclDenseRM(ompC)
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP multiplication time: $ms ms.")
+
+ ompA.close()
+ ompB.close()
+ ompC.close()
+ }
+ oclCtx.deallocate()
+ oclCtx.close()
+
+
+ }
+
+ test("trans") {
+
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val ompCtx = new Context(Context.MAIN_MEMORY)
+
+
+ val m = 20
+ val n = 30
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, n)
+
+ // add some data
+ mxA := { (_, _, _) => r.nextDouble() }
+
+ // Test transposition in OpenCL
+ {
+ val oclA = toVclDenseRM(src = mxA, oclCtx)
+ val oclAt = new DenseRowMatrix(trans(oclA))
+
+ val mxAt = fromVclDenseRM(oclAt)
+ oclA.close()
+ oclAt.close()
+
+ (mxAt - mxA.t).norm / m / n should be < 1e-16
+ }
+
+ // Test transposition in OpenMP
+ {
+ val ompA = toVclDenseRM(src = mxA, ompCtx)
+ val ompAt = new DenseRowMatrix(trans(ompA))
+
+ val mxAt = fromVclDenseRM(ompAt)
+ ompA.close()
+ ompAt.close()
+
+ (mxAt - mxA.t).norm / m / n should be < 1e-16
+ }
+ oclCtx.deallocate()
+ oclCtx.close()
+
+
+ }
+
+ test("sparse mmul microbenchmark") {
+
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val ompCtx = new Context(Context.MAIN_MEMORY)
+
+
+ val m = 3000
+ val n = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // sparse row-wise
+ val mxA = new SparseRowMatrix(m, s, false)
+ val mxB = new SparseRowMatrix(s, n, true)
+
+ // add some sparse data with a 20% threshold
+ mxA := { (_, _, v) => if (r.nextDouble() < .20) r.nextDouble() else v }
+ mxB := { (_, _, v) => if (r.nextDouble() < .20) r.nextDouble() else v }
+
+ var ms = System.currentTimeMillis()
+ val mxC = mxA %*% mxB
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout Sparse multiplication time: $ms ms.")
+
+// Test multiplication in OpenCL
+ {
+
+ ms = System.currentTimeMillis()
+ val oclA = toVclCmpMatrixAlt(mxA, oclCtx)
+ val oclB = toVclCmpMatrixAlt(mxB, oclCtx)
+
+ val oclC = new CompressedMatrix(prod(oclA, oclB))
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/OpenCL Sparse multiplication time: $ms ms.")
+
+ val oclMxC = fromVclCompressedMatrix(oclC)
+ val ompMxC = fromVclCompressedMatrix(oclC)
+ (mxC - oclMxC).norm / mxC.nrow / mxC.ncol should be < 1e-16
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+ }
+
+ // Test multiplication in OpenMP
+ {
+ ms = System.currentTimeMillis()
+ // val ompA = toVclCompressedMatrix(src = mxA, ompCtx)
+ // val ompB = toVclCompressedMatrix(src = mxB, ompCtx)
+
+ val ompA = toVclCmpMatrixAlt(mxA, ompCtx)
+ val ompB = toVclCmpMatrixAlt(mxB, ompCtx)
+
+ val ompC = new CompressedMatrix(prod(ompA, ompB))
+
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP Sparse multiplication time: $ms ms.")
+
+ val ompMxC = fromVclCompressedMatrix(ompC)
+ (mxC - ompMxC).norm / mxC.nrow / mxC.ncol should be < 1e-16
+
+ ompA.close()
+ ompB.close()
+ ompC.close()
+
+ }
+ oclCtx.deallocate()
+ oclCtx.close()
+
+ }
+
+ test("VCL Dense Matrix %*% Dense vector") {
+
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val ompCtx = new Context(Context.MAIN_MEMORY)
+
+
+ val m = 30
+ val s = 10
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, s)
+ val dvecB = new DenseVector(s)
+
+ // add some random data
+ mxA := { (_,_,_) => r.nextDouble() }
+ dvecB := { (_,_) => r.nextDouble() }
+
+ //test in matrix %*% vec
+ var ms = System.currentTimeMillis()
+ val mDvecC = mxA %*% dvecB
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout dense matrix %*% dense vector multiplication time: $ms ms.")
+
+
+ /* TODO: CL_OUT_OF_RESOURCES error thrown when trying to read data out of OpenCl GPU Vectors */
+ //Test multiplication in OpenCL
+// {
+//
+// ms = System.currentTimeMillis()
+// val oclA = toVclDenseRM(mxA, oclCtx)
+// val oclVecB = toVclVec(dvecB, oclCtx)
+//
+// val oclVecC = new VCLVector(prod(oclA, oclVecB))
+// val oclDvecC = fromVClVec(oclVecC)
+////
+//// ms = System.currentTimeMillis() - ms
+//// info(s"ViennaCL/OpenCL dense matrix %*% dense vector multiplication time: $ms ms.")
+//// (oclDvecC.toColMatrix - mDvecC.toColMatrix).norm / s should be < 1e-16
+//
+// oclA.close()
+// oclVecB.close()
+// oclVecC.close()
+// }
+
+ //Test multiplication in OpenMP
+ {
+
+ ms = System.currentTimeMillis()
+ val ompMxA = toVclDenseRM(mxA, ompCtx)
+ val ompVecB = toVclVec(dvecB, ompCtx)
+
+ val ompVecC = new VCLVector(prod(ompMxA, ompVecB))
+ val ompDvecC = fromVClVec(ompVecC)
+
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP dense matrix %*% dense vector multiplication time: $ms ms.")
+ (ompDvecC.toColMatrix - mDvecC.toColMatrix).norm / s should be < 1e-16
+
+ ompMxA.close()
+ ompVecB.close()
+ ompVecC.close()
+ }
+
+ oclCtx.deallocate()
+ oclCtx.close()
+
+
+ }
+
+
+ test("Sparse %*% Dense mmul microbenchmark") {
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val memCtx = new Context(Context.MAIN_MEMORY)
+
+ val m = 3000
+ val n = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxSr = new SparseMatrix(m, s)
+ val mxDn = new DenseMatrix(s, n)
+
+ // add some data
+ mxSr := { (_, _, v) => if (r.nextDouble() < .20) r.nextDouble() else v }
+ mxDn := { (_, _, _) => r.nextDouble() }
+
+ var ms = System.currentTimeMillis()
+ mxSr %*% mxDn
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout multiplication time: $ms ms.")
+
+ import LinalgFunctions._
+
+ // For now, since our dense matrix is fully dense lets just assume that our result is dense.
+ // openCL time, including copying:
+ {
+ ms = System.currentTimeMillis()
+ val oclA = toVclCmpMatrixAlt(mxSr, oclCtx)
+ val oclB = toVclDenseRM(mxDn, oclCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+ }
+
+ // openMP/cpu time, including copying:
+ {
+ ms = System.currentTimeMillis()
+ val ompA = toVclCmpMatrixAlt(mxSr, memCtx)
+ val ompB = toVclDenseRM(mxDn, memCtx)
+ val ompC = new DenseRowMatrix(prod(ompA, ompB))
+ val mxC = fromVclDenseRM(ompC)
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP multiplication time: $ms ms.")
+
+ ompA.close()
+ ompB.close()
+ ompC.close()
+ }
+
+ oclCtx.deallocate()
+ oclCtx.close()
+
+
+ }
+
+
+
+}
[5/5] mahout git commit: MAHOUT-1885: Inital commit of VCL bindings.
closes apache/mahout#269 closes apache/mahout#261
Posted by ap...@apache.org.
MAHOUT-1885: Inital commit of VCL bindings. closes apache/mahout#269 closes apache/mahout#261
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/034790cc
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/034790cc
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/034790cc
Branch: refs/heads/master
Commit: 034790cce40fcee2b7a875c482345d35f7c0fa8d
Parents: 84e90ed
Author: Andrew Palumbo <ap...@apache.org>
Authored: Wed Jan 25 20:23:28 2017 -0800
Committer: Andrew Palumbo <ap...@apache.org>
Committed: Wed Jan 25 20:23:28 2017 -0800
----------------------------------------------------------------------
.travis.yml | 7 +-
LICENSE.txt | 605 +++++++
bin/mahout | 26 +
distribution/pom.xml | 109 ++
examples/bin/SparseSparseDrmTimer.mscala | 37 +
flink/pom.xml | 6 +
math-scala/pom.xml | 10 +-
.../apache/mahout/math/backend/Backend.scala | 33 +
.../mahout/math/backend/RootSolverFactory.scala | 87 +
.../mahout/math/backend/SolverFactory.scala | 55 +
.../mahout/math/backend/incore/package.scala | 17 +
.../mahout/math/backend/jvm/JvmBackend.scala | 51 +
.../apache/mahout/math/scalabindings/MMul.scala | 3 +-
.../math/scalabindings/RLikeMatrixOps.scala | 14 +-
.../mahout/math/scalabindings/package.scala | 3 +
.../mahout/math/backend/BackendSuite.scala | 59 +
.../scalabindings/RLikeMatrixOpsSuite.scala | 3 +
pom.xml | 37 +-
runtests.sh | 1 +
spark-shell/pom.xml | 6 +
.../sparkbindings/shell/MahoutSparkILoop.scala | 5 +-
spark/pom.xml | 6 +
spark/src/main/assembly/dependency-reduced.xml | 2 +
.../apache/mahout/sparkbindings/blas/ABt.scala | 2 +-
.../apache/mahout/sparkbindings/package.scala | 33 +-
.../drivers/ItemSimilarityDriverSuite.scala | 1664 +++++++++---------
.../drivers/RowSimilarityDriverSuite.scala | 278 +--
.../TextDelimitedReaderWriterSuite.scala | 106 +-
.../sparkbindings/SparkBindingsSuite.scala | 24 +-
.../test/DistributedSparkSuite.scala | 2 +-
viennacl-omp/linux-haswell.properties | 28 +
viennacl-omp/linux-x86_64-viennacl.properties | 24 +
viennacl-omp/pom.xml | 278 +++
viennacl-omp/runs | 32 +
.../viennacl/openmp/javacpp/Functions.java | 103 ++
.../openmp/javacpp/LinalgFunctions.java | 86 +
.../openmp/javacpp/MatrixTransExpression.scala | 34 +
.../apache/mahout/viennacl/openmp/OMPMMul.scala | 449 +++++
.../openmp/javacpp/CompressedMatrix.scala | 125 ++
.../viennacl/openmp/javacpp/Context.scala | 58 +
.../openmp/javacpp/DenseColumnMatrix.scala | 83 +
.../openmp/javacpp/DenseRowMatrix.scala | 69 +
.../openmp/javacpp/MatMatProdExpression.scala | 33 +
.../openmp/javacpp/MatVecProdExpression.scala | 33 +
.../viennacl/openmp/javacpp/MatrixBase.scala | 75 +
.../viennacl/openmp/javacpp/MemHandle.scala | 34 +
.../openmp/javacpp/ProdExpression.scala | 33 +
.../javacpp/SrMatDnMatProdExpression.scala | 33 +
.../viennacl/openmp/javacpp/VCLVector.scala | 115 ++
.../openmp/javacpp/VecMultExpression.scala | 32 +
.../viennacl/openmp/javacpp/VectorBase.scala | 55 +
.../apache/mahout/viennacl/openmp/package.scala | 434 +++++
.../mahout/viennacl/omp/ViennaCLSuiteOMP.scala | 249 +++
viennacl/linux-haswell.properties | 28 +
viennacl/linux-x86_64-viennacl.properties | 24 +
viennacl/pom.xml | 271 +++
.../viennacl/opencl/javacpp/Functions.java | 104 ++
.../opencl/javacpp/LinalgFunctions.java | 86 +
.../opencl/javacpp/MatrixTransExpression.scala | 34 +
.../apache/mahout/viennacl/opencl/GPUMMul.scala | 455 +++++
.../opencl/javacpp/CompressedMatrix.scala | 125 ++
.../viennacl/opencl/javacpp/Context.scala | 73 +
.../opencl/javacpp/DenseColumnMatrix.scala | 83 +
.../opencl/javacpp/DenseRowMatrix.scala | 86 +
.../opencl/javacpp/MatMatProdExpression.scala | 33 +
.../opencl/javacpp/MatVecProdExpression.scala | 33 +
.../viennacl/opencl/javacpp/MatrixBase.scala | 75 +
.../viennacl/opencl/javacpp/MemHandle.scala | 48 +
.../opencl/javacpp/ProdExpression.scala | 33 +
.../javacpp/SrMatDnMatProdExpression.scala | 33 +
.../viennacl/opencl/javacpp/VCLVector.scala | 133 ++
.../opencl/javacpp/VecMultExpression.scala | 32 +
.../viennacl/opencl/javacpp/VectorBase.scala | 57 +
.../apache/mahout/viennacl/opencl/package.scala | 434 +++++
.../viennacl/opencl/ViennaCLSuiteVCL.scala | 427 +++++
75 files changed, 7428 insertions(+), 1065 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index 426d57e..36653d7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,4 +1,4 @@
-sudo: false
+sudo: required
cache:
directories:
@@ -33,8 +33,11 @@ before_install:
- unzip -qq apache-maven-3.3.9-bin.zip
- export M2_HOME=$PWD/apache-maven-3.3.9
- export PATH=$M2_HOME/bin:$PATH
+ - export MAHOUT_HOME=$PWD
+ - sudo apt-get -qq update
+ - sudo apt-get install -y libviennacl-dev
script:
- ./runtests.sh
-- mvn javadoc:javadoc
+#- mvn javadoc:javadoc
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/LICENSE.txt
----------------------------------------------------------------------
diff --git a/LICENSE.txt b/LICENSE.txt
index 8ce7fff..110fb6e 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -799,3 +799,608 @@ The following license applies to the H2O package
identification within third-party archives.
Copyright 2012 0xdata, Inc
+
+================================================================
+The following applies to the ViennaCL library and files in the mahout-native-viennacl module
+================================================================
+
+ Copyright (c) 2010-2016 Institute for Microelectronics,
+ Institute for Analysis and Scientific Computing, TU Wien.
+ Portions of this software are copyright by UChicago Argonne, LLC.
+ Argonne National Laboratory, with facilities in the state of Illinois,
+ is owned by The United States Government, and operated by UChicago Argonne, LLC
+ under provision of a contract with the Department of Energy.
+
+ Permission is hereby granted, free of charge, to any person obtaining a copy
+ of this software and associated documentation files (the "Software"), to deal
+ in the Software without restriction, including without limitation the rights
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ copies of the Software, and to permit persons to whom the Software is
+ furnished to do so, subject to the following conditions:
+
+ The above copyright notice and this permission notice shall be included in
+ all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ THE SOFTWARE.
+
+ ===============================================================================
+ The following applies to the bydeco/javacpp code and module
+ ===============================================================================
+ You may use this work under the terms of either the Apache License,
+ Version 2.0, or the GNU General Public License (GPL), either version 2,
+ or any later version, with "Classpath" exception (details below).
+
+ You don't have to do anything special to choose one license or the other
+ and you don't have to notify anyone which license you are using. You are
+ free to use this work in any project (even commercial projects) as long
+ as the copyright header is left intact.
+
+ ===============================================================================
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ ===============================================================================
+
+ GNU GENERAL PUBLIC LICENSE
+ Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The licenses for most software are designed to take away your
+ freedom to share and change it. By contrast, the GNU General Public
+ License is intended to guarantee your freedom to share and change free
+ software--to make sure the software is free for all its users. This
+ General Public License applies to most of the Free Software
+ Foundation's software and to any other program whose authors commit to
+ using it. (Some other Free Software Foundation software is covered by
+ the GNU Lesser General Public License instead.) You can apply it to
+ your programs, too.
+
+ When we speak of free software, we are referring to freedom, not
+ price. Our General Public Licenses are designed to make sure that you
+ have the freedom to distribute copies of free software (and charge for
+ this service if you wish), that you receive source code or can get it
+ if you want it, that you can change the software or use pieces of it
+ in new free programs; and that you know you can do these things.
+
+ To protect your rights, we need to make restrictions that forbid
+ anyone to deny you these rights or to ask you to surrender the rights.
+ These restrictions translate to certain responsibilities for you if you
+ distribute copies of the software, or if you modify it.
+
+ For example, if you distribute copies of such a program, whether
+ gratis or for a fee, you must give the recipients all the rights that
+ you have. You must make sure that they, too, receive or can get the
+ source code. And you must show them these terms so they know their
+ rights.
+
+ We protect your rights with two steps: (1) copyright the software, and
+ (2) offer you this license which gives you legal permission to copy,
+ distribute and/or modify the software.
+
+ Also, for each author's protection and ours, we want to make certain
+ that everyone understands that there is no warranty for this free
+ software. If the software is modified by someone else and passed on, we
+ want its recipients to know that what they have is not the original, so
+ that any problems introduced by others will not reflect on the original
+ authors' reputations.
+
+ Finally, any free program is threatened constantly by software
+ patents. We wish to avoid the danger that redistributors of a free
+ program will individually obtain patent licenses, in effect making the
+ program proprietary. To prevent this, we have made it clear that any
+ patent must be licensed for everyone's free use or not licensed at all.
+
+ The precise terms and conditions for copying, distribution and
+ modification follow.
+
+ GNU GENERAL PUBLIC LICENSE
+ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+ 0. This License applies to any program or other work which contains
+ a notice placed by the copyright holder saying it may be distributed
+ under the terms of this General Public License. The "Program", below,
+ refers to any such program or work, and a "work based on the Program"
+ means either the Program or any derivative work under copyright law:
+ that is to say, a work containing the Program or a portion of it,
+ either verbatim or with modifications and/or translated into another
+ language. (Hereinafter, translation is included without limitation in
+ the term "modification".) Each licensee is addressed as "you".
+
+ Activities other than copying, distribution and modification are not
+ covered by this License; they are outside its scope. The act of
+ running the Program is not restricted, and the output from the Program
+ is covered only if its contents constitute a work based on the
+ Program (independent of having been made by running the Program).
+ Whether that is true depends on what the Program does.
+
+ 1. You may copy and distribute verbatim copies of the Program's
+ source code as you receive it, in any medium, provided that you
+ conspicuously and appropriately publish on each copy an appropriate
+ copyright notice and disclaimer of warranty; keep intact all the
+ notices that refer to this License and to the absence of any warranty;
+ and give any other recipients of the Program a copy of this License
+ along with the Program.
+
+ You may charge a fee for the physical act of transferring a copy, and
+ you may at your option offer warranty protection in exchange for a fee.
+
+ 2. You may modify your copy or copies of the Program or any portion
+ of it, thus forming a work based on the Program, and copy and
+ distribute such modifications or work under the terms of Section 1
+ above, provided that you also meet all of these conditions:
+
+ a) You must cause the modified files to carry prominent notices
+ stating that you changed the files and the date of any change.
+
+ b) You must cause any work that you distribute or publish, that in
+ whole or in part contains or is derived from the Program or any
+ part thereof, to be licensed as a whole at no charge to all third
+ parties under the terms of this License.
+
+ c) If the modified program normally reads commands interactively
+ when run, you must cause it, when started running for such
+ interactive use in the most ordinary way, to print or display an
+ announcement including an appropriate copyright notice and a
+ notice that there is no warranty (or else, saying that you provide
+ a warranty) and that users may redistribute the program under
+ these conditions, and telling the user how to view a copy of this
+ License. (Exception: if the Program itself is interactive but
+ does not normally print such an announcement, your work based on
+ the Program is not required to print an announcement.)
+
+ These requirements apply to the modified work as a whole. If
+ identifiable sections of that work are not derived from the Program,
+ and can be reasonably considered independent and separate works in
+ themselves, then this License, and its terms, do not apply to those
+ sections when you distribute them as separate works. But when you
+ distribute the same sections as part of a whole which is a work based
+ on the Program, the distribution of the whole must be on the terms of
+ this License, whose permissions for other licensees extend to the
+ entire whole, and thus to each and every part regardless of who wrote it.
+
+ Thus, it is not the intent of this section to claim rights or contest
+ your rights to work written entirely by you; rather, the intent is to
+ exercise the right to control the distribution of derivative or
+ collective works based on the Program.
+
+ In addition, mere aggregation of another work not based on the Program
+ with the Program (or with a work based on the Program) on a volume of
+ a storage or distribution medium does not bring the other work under
+ the scope of this License.
+
+ 3. You may copy and distribute the Program (or a work based on it,
+ under Section 2) in object code or executable form under the terms of
+ Sections 1 and 2 above provided that you also do one of the following:
+
+ a) Accompany it with the complete corresponding machine-readable
+ source code, which must be distributed under the terms of Sections
+ 1 and 2 above on a medium customarily used for software interchange; or,
+
+ b) Accompany it with a written offer, valid for at least three
+ years, to give any third party, for a charge no more than your
+ cost of physically performing source distribution, a complete
+ machine-readable copy of the corresponding source code, to be
+ distributed under the terms of Sections 1 and 2 above on a medium
+ customarily used for software interchange; or,
+
+ c) Accompany it with the information you received as to the offer
+ to distribute corresponding source code. (This alternative is
+ allowed only for noncommercial distribution and only if you
+ received the program in object code or executable form with such
+ an offer, in accord with Subsection b above.)
+
+ The source code for a work means the preferred form of the work for
+ making modifications to it. For an executable work, complete source
+ code means all the source code for all modules it contains, plus any
+ associated interface definition files, plus the scripts used to
+ control compilation and installation of the executable. However, as a
+ special exception, the source code distributed need not include
+ anything that is normally distributed (in either source or binary
+ form) with the major components (compiler, kernel, and so on) of the
+ operating system on which the executable runs, unless that component
+ itself accompanies the executable.
+
+ If distribution of executable or object code is made by offering
+ access to copy from a designated place, then offering equivalent
+ access to copy the source code from the same place counts as
+ distribution of the source code, even though third parties are not
+ compelled to copy the source along with the object code.
+
+ 4. You may not copy, modify, sublicense, or distribute the Program
+ except as expressly provided under this License. Any attempt
+ otherwise to copy, modify, sublicense or distribute the Program is
+ void, and will automatically terminate your rights under this License.
+ However, parties who have received copies, or rights, from you under
+ this License will not have their licenses terminated so long as such
+ parties remain in full compliance.
+
+ 5. You are not required to accept this License, since you have not
+ signed it. However, nothing else grants you permission to modify or
+ distribute the Program or its derivative works. These actions are
+ prohibited by law if you do not accept this License. Therefore, by
+ modifying or distributing the Program (or any work based on the
+ Program), you indicate your acceptance of this License to do so, and
+ all its terms and conditions for copying, distributing or modifying
+ the Program or works based on it.
+
+ 6. Each time you redistribute the Program (or any work based on the
+ Program), the recipient automatically receives a license from the
+ original licensor to copy, distribute or modify the Program subject to
+ these terms and conditions. You may not impose any further
+ restrictions on the recipients' exercise of the rights granted herein.
+ You are not responsible for enforcing compliance by third parties to
+ this License.
+
+ 7. If, as a consequence of a court judgment or allegation of patent
+ infringement or for any other reason (not limited to patent issues),
+ conditions are imposed on you (whether by court order, agreement or
+ otherwise) that contradict the conditions of this License, they do not
+ excuse you from the conditions of this License. If you cannot
+ distribute so as to satisfy simultaneously your obligations under this
+ License and any other pertinent obligations, then as a consequence you
+ may not distribute the Program at all. For example, if a patent
+ license would not permit royalty-free redistribution of the Program by
+ all those who receive copies directly or indirectly through you, then
+ the only way you could satisfy both it and this License would be to
+ refrain entirely from distribution of the Program.
+
+ If any portion of this section is held invalid or unenforceable under
+ any particular circumstance, the balance of the section is intended to
+ apply and the section as a whole is intended to apply in other
+ circumstances.
+
+ It is not the purpose of this section to induce you to infringe any
+ patents or other property right claims or to contest validity of any
+ such claims; this section has the sole purpose of protecting the
+ integrity of the free software distribution system, which is
+ implemented by public license practices. Many people have made
+ generous contributions to the wide range of software distributed
+ through that system in reliance on consistent application of that
+ system; it is up to the author/donor to decide if he or she is willing
+ to distribute software through any other system and a licensee cannot
+ impose that choice.
+
+ This section is intended to make thoroughly clear what is believed to
+ be a consequence of the rest of this License.
+
+ 8. If the distribution and/or use of the Program is restricted in
+ certain countries either by patents or by copyrighted interfaces, the
+ original copyright holder who places the Program under this License
+ may add an explicit geographical distribution limitation excluding
+ those countries, so that distribution is permitted only in or among
+ countries not thus excluded. In such case, this License incorporates
+ the limitation as if written in the body of this License.
+
+ 9. The Free Software Foundation may publish revised and/or new versions
+ of the General Public License from time to time. Such new versions will
+ be similar in spirit to the present version, but may differ in detail to
+ address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the Program
+ specifies a version number of this License which applies to it and "any
+ later version", you have the option of following the terms and conditions
+ either of that version or of any later version published by the Free
+ Software Foundation. If the Program does not specify a version number of
+ this License, you may choose any version ever published by the Free Software
+ Foundation.
+
+ 10. If you wish to incorporate parts of the Program into other free
+ programs whose distribution conditions are different, write to the author
+ to ask for permission. For software which is copyrighted by the Free
+ Software Foundation, write to the Free Software Foundation; we sometimes
+ make exceptions for this. Our decision will be guided by the two goals
+ of preserving the free status of all derivatives of our free software and
+ of promoting the sharing and reuse of software generally.
+
+ NO WARRANTY
+
+ 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+ FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN
+ OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+ PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+ OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS
+ TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE
+ PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+ REPAIR OR CORRECTION.
+
+ 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+ WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+ REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+ INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+ OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+ TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGES.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+ possible use to the public, the best way to achieve this is to make it
+ free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+ to attach them to the start of each source file to most effectively
+ convey the exclusion of warranty; and each file should have at least
+ the "copyright" line and a pointer to where the full notice is found.
+
+ <one line to give the program's name and a brief idea of what it does.>
+ Copyright (C) <year> <name of author>
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with this program; if not, write to the Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+ Also add information on how to contact you by electronic and paper mail.
+
+ If the program is interactive, make it output a short notice like this
+ when it starts in an interactive mode:
+
+ Gnomovision version 69, Copyright (C) year name of author
+ Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+ This is free software, and you are welcome to redistribute it
+ under certain conditions; type `show c' for details.
+
+ The hypothetical commands `show w' and `show c' should show the appropriate
+ parts of the General Public License. Of course, the commands you use may
+ be called something other than `show w' and `show c'; they could even be
+ mouse-clicks or menu items--whatever suits your program.
+
+ You should also get your employer (if you work as a programmer) or your
+ school, if any, to sign a "copyright disclaimer" for the program, if
+ necessary. Here is a sample; alter the names:
+
+ Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+ `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+ <signature of Ty Coon>, 1 April 1989
+ Ty Coon, President of Vice
+
+ This General Public License does not permit incorporating your program into
+ proprietary programs. If your program is a subroutine library, you may
+ consider it more useful to permit linking proprietary applications with the
+ library. If this is what you want to do, use the GNU Lesser General
+ Public License instead of this License.
+
+
+ "CLASSPATH" EXCEPTION TO THE GPL
+
+ Linking this library statically or dynamically with other modules is making
+ a combined work based on this library. Thus, the terms and conditions of
+ the GNU General Public License cover the whole combination.
+
+ As a special exception, the copyright holders of this library give you
+ permission to link this library with independent modules to produce an
+ executable, regardless of the license terms of these independent modules,
+ and to copy and distribute the resulting executable under terms of your
+ choice, provided that you also meet, for each linked independent module,
+ the terms and conditions of the license of that module. An independent
+ module is a module which is not derived from or based on this library. If
+ you modify this library, you may extend this exception to your version of
+ the library, but you are not obligated to do so. If you do not wish to do
+ so, delete this exception statement from your version.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/bin/mahout
----------------------------------------------------------------------
diff --git a/bin/mahout b/bin/mahout
index c58e865..d88abe8 100755
--- a/bin/mahout
+++ b/bin/mahout
@@ -211,6 +211,21 @@ then
CLASSPATH=${CLASSPATH}:$f;
done
+ # viennacl jars- may or may not be available depending on build profile
+ for f in $MAHOUT_HOME/viennacl/target/mahout-native-viennacl_*.jar ; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
+ # viennacl jars- may or may not be available depending on build profile
+ for f in $MAHOUT_HOME/viennacl-omp/target/mahout-native-viennacl-omp_*.jar ; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
+ # viennacl jars- may or may not be available depending on build profile
+ for f in $MAHOUT_HOME/viennacl-omp/target/mahout-native-viennacl-omp_*.jar ; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
SPARK_CP_BIN="${MAHOUT_HOME}/bin/compute-classpath.sh"
if [ -x "${SPARK_CP_BIN}" ]; then
SPARK_CLASSPATH=$("${SPARK_CP_BIN}" 2>/dev/null)
@@ -230,6 +245,17 @@ then
fi
fi
+ # add vcl jars at any point.
+ # viennacl jars- may or may not be available depending on build profile
+ for f in $MAHOUT_HOME/viennacl/target/mahout-native-viennacl_*.jar ; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
+ # viennacl jars- may or may not be available depending on build profile
+ for f in $MAHOUT_HOME/viennacl-omp/target/mahout-native-viennacl-omp_*.jar ; do
+ CLASSPATH=${CLASSPATH}:$f;
+ done
+
# add release dependencies to CLASSPATH
for f in $MAHOUT_HOME/lib/*.jar; do
CLASSPATH=${CLASSPATH}:$f;
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/distribution/pom.xml
----------------------------------------------------------------------
diff --git a/distribution/pom.xml b/distribution/pom.xml
index 536c76f..b8295ba 100644
--- a/distribution/pom.xml
+++ b/distribution/pom.xml
@@ -78,6 +78,106 @@
<mahout.skip.distribution>false</mahout.skip.distribution>
</properties>
</profile>
+ <profile>
+ <id>viennacl</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-math</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-integration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-mr</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-examples</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-spark_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-flink_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-spark-shell_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-math-scala_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId>
+ <version>0.13.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>viennacl-omp</id>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-math</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-integration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-mr</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-examples</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-spark_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-flink_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-spark-shell_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-math-scala_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId>
+ <version>0.13.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+ </profile>
+
+
+
</profiles>
<dependencies>
@@ -117,5 +217,14 @@
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-math-scala_${scala.compat.version}</artifactId>
</dependency>
+ <!--Viennacl is not part of the Default build currently.-->
+ <!--<dependency>-->
+ <!--<groupId>org.apache.mahout</groupId>-->
+ <!--<artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId>-->
+ <!--</dependency>-->
+ <!--<dependency>-->
+ <!--<groupId>org.apache.mahout</groupId>-->
+ <!--<artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId>-->
+ <!--</dependency>-->
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/examples/bin/SparseSparseDrmTimer.mscala
----------------------------------------------------------------------
diff --git a/examples/bin/SparseSparseDrmTimer.mscala b/examples/bin/SparseSparseDrmTimer.mscala
new file mode 100644
index 0000000..d01bf3a
--- /dev/null
+++ b/examples/bin/SparseSparseDrmTimer.mscala
@@ -0,0 +1,37 @@
+
+
+def timeSparseDRMMMul(m: Int, n: Int, s: Int, para: Int, pctDense: Double = .20, seed: Long = 1234L): Long = {
+
+
+
+ val drmA = drmParallelizeEmpty(m , s, para).mapBlock(){
+ case (keys,block:Matrix) =>
+ val R = scala.util.Random
+ R.setSeed(seed)
+ val blockB = new SparseRowMatrix(block.nrow, block.ncol)
+ blockB := {x => if (R.nextDouble > pctDense) R.nextDouble else x }
+ (keys -> blockB)
+ }
+ val drmB = drmParallelizeEmpty(s , n, para).mapBlock(){
+ case (keys,block:Matrix) =>
+ val R = scala.util.Random
+ R.setSeed(seed + 1)
+ val blockB = new SparseRowMatrix(block.nrow, block.ncol)
+ blockB := {x => if (R.nextDouble > pctDense) R.nextDouble else x }
+ (keys -> blockB)
+ }
+
+ var time = System.currentTimeMillis()
+
+ val drmC = drmA %*% drmB
+
+ // trigger computation
+ drmC.numRows()
+
+ time = System.currentTimeMillis() - time
+
+ time
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/flink/pom.xml
----------------------------------------------------------------------
diff --git a/flink/pom.xml b/flink/pom.xml
index 7857210..f9ec56b 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -153,6 +153,12 @@
<artifactId>mahout-math-scala_${scala.compat.version}</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ <version>1.2.2</version>
+ </dependency>
+
<!-- enforce current version of kryo as of 0.10.1-->
<dependency>
<groupId>com.esotericsoftware.kryo</groupId>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/pom.xml
----------------------------------------------------------------------
diff --git a/math-scala/pom.xml b/math-scala/pom.xml
index 9eb7e80..da1c05b 100644
--- a/math-scala/pom.xml
+++ b/math-scala/pom.xml
@@ -102,6 +102,9 @@
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
+ <configuration>
+ <argLine>-Xmx4g</argLine>
+ </configuration>
<executions>
<execution>
<id>test</id>
@@ -147,10 +150,13 @@
<artifactId>scalatest_${scala.compat.version}</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ <version>${scala.version}</version>
+ </dependency>
</dependencies>
-
<profiles>
<profile>
<id>mahout-release</id>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala
new file mode 100644
index 0000000..9dfb7f2
--- /dev/null
+++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/Backend.scala
@@ -0,0 +1,33 @@
+package org.apache.mahout.math.backend
+
+import org.apache.mahout.math.backend.jvm.JvmBackend
+
+import collection._
+import scala.reflect.{ClassTag, classTag}
+import jvm.JvmBackend
+
+/**
+ * == Overview ==
+ *
+ * Backend representing collection of in-memory solvers or distributed operators.
+ *
+ * == Note to implementors ==
+ *
+ * Backend is expected to initialize & verify its own viability lazily either upon first time the
+ * class is loaded, or upon the first invocation of any of its methods. After that, the value of
+ * [[Backend.isAvailable]] must be cached and defined.
+ *
+ * A Backend is also a [[SolverFactory]] of course in a sense that it enumerates solvers made
+ * available via the backend.
+ */
+trait Backend extends SolverFactory {
+
+ /**
+ * If backend has loaded (lazily) ok and verified its availability/functionality,
+ * this must return `true`.
+ *
+ * @return `true`
+ */
+ def isAvailable: Boolean
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala
new file mode 100644
index 0000000..253a435
--- /dev/null
+++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/RootSolverFactory.scala
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.math.backend
+
+import org.apache.mahout.logging._
+import org.apache.mahout.math.backend.jvm.JvmBackend
+import org.apache.mahout.math.scalabindings.{MMul, _}
+
+import scala.collection._
+import scala.reflect.{ClassTag, classTag}
+
+
+final object RootSolverFactory extends SolverFactory {
+
+ import org.apache.mahout.math.backend.incore._
+
+ implicit val logger = getLog(RootSolverFactory.getClass)
+
+ private val solverTagsToScan =
+ classTag[MMulSolver] ::
+ classTag[MMulSparseSolver] ::
+ classTag[MMulDenseSolver] ::
+ Nil
+
+ private val defaultBackendPriority =
+ JvmBackend.getClass.getName :: Nil
+
+ private def initBackends(): Unit = {
+
+ }
+
+ ////////////////////////////////////////////////////////////
+
+ // TODO: MAHOUT-1909: lazy initialze the map. Query backends. Build resolution rules.
+ override protected[backend] val solverMap = new mutable.HashMap[ClassTag[_], Any]()
+ validateMap()
+
+
+ // default is JVM
+ var clazz: MMBinaryFunc = MMul
+
+ // eventually match on implicit Classtag . for now. just take as is.
+ // this is a bit hacky, Shoud not be doing onlytry/catch here..
+ def getOperator[C: ClassTag]: MMBinaryFunc = {
+
+ try {
+ // TODO: fix logging properties so that we're not mimicing as we are here.
+ println("[INFO] Creating org.apache.mahout.viennacl.opencl.GPUMMul solver")
+ clazz = Class.forName("org.apache.mahout.viennacl.opencl.GPUMMul$").getField("MODULE$").get(null).asInstanceOf[MMBinaryFunc]
+ println("[INFO] Successfully created org.apache.mahout.viennacl.opencl.GPUMMul solver")
+
+ } catch {
+ case x: Exception =>
+ println("[WARN] Unable to create class GPUMMul: attempting OpenMP version")
+ // println(x.getMessage)
+ try {
+ // attempt to instantiate the OpenMP version, assuming we\u2019ve
+ // created a separate OpenMP-only module (none exist yet)
+ println("[INFO] Creating org.apache.mahout.viennacl.openmp.OMPMMul solver")
+ clazz = Class.forName("org.apache.mahout.viennacl.openmp.OMPMMul$").getField("MODULE$").get(null).asInstanceOf[MMBinaryFunc]
+ println("[INFO] Successfully created org.apache.mahout.viennacl.openmp.OMPMMul solver")
+
+ } catch {
+ case xx: Exception =>
+ println(xx.getMessage)
+ // fall back to JVM Dont need to Dynamicly assign MMul is in the same package.
+ println("[INFO] Unable to create class OMPMMul: falling back to java version")
+ clazz = MMul
+ }
+ }
+ clazz
+ }
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala
new file mode 100644
index 0000000..756b971
--- /dev/null
+++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/SolverFactory.scala
@@ -0,0 +1,55 @@
+package org.apache.mahout.math.backend
+
+import scala.collection.{Iterable, Map}
+import scala.reflect.{ClassTag, classTag}
+
+/**
+ * == Overview ==
+ *
+ * Solver factory is an essence a collection of lazily initialized strategy singletons solving some
+ * (any) problem in context of the Mahout project.
+ *
+ * We intend to use it _mainly_ for problems that are super-linear problems, and often involve more
+ * than one argument (operand).
+ *
+ * The main method to probe for an available solver is [[RootSolverFactory.getSolver]].
+ */
+trait SolverFactory {
+ /**
+ * We take an implicit context binding, the classTag, of the trait of the solver desired.
+ *
+ * == Note to callers ==
+ *
+ * Due to Scala semantics, it is usually not enough to request a solver via merely {{{
+ * val s:SolverType = backend.getSolver
+ * }}} but instead requires an explicit solver tag, i.e.: {{{
+ * val s = backend.getSolver[SolverType]
+ * }}}
+ *
+ *
+ */
+ def getSolver[S: ClassTag]: Option[S] = {
+ solverMap.get(classTag[S]).flatMap {
+ _ match {
+ case s: S \u21d2 Some(s)
+ case _ \u21d2 None
+ }
+ }
+ }
+
+ lazy val availableSolverTags: Iterable[ClassTag[_]] = solverMap.keySet
+
+
+
+ protected[backend] val solverMap: Map[ClassTag[_], Any]
+
+ protected[backend] def validateMap(): Unit = {
+
+ for ((tag, instance) \u2190 solverMap) {
+ require(tag.runtimeClass.isAssignableFrom(instance.getClass),
+ s"Solver implementation class `${instance.getClass.getName}` is not a subclass of solver trait `${tag}`.")
+
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala
new file mode 100644
index 0000000..1bb4480
--- /dev/null
+++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/incore/package.scala
@@ -0,0 +1,17 @@
+package org.apache.mahout.math.backend
+
+import org.apache.mahout.math.scalabindings.{MMBinaryFunc, MMUnaryFunc}
+
+package object incore {
+
+ trait MMulSolver extends MMBinaryFunc
+ trait MMulDenseSolver extends MMulSolver
+ trait MMulSparseSolver extends MMulSolver
+ trait AAtSolver extends MMUnaryFunc
+ trait AAtDenseSolver extends AAtSolver
+ trait AAtSparseSolver extends AAtSolver
+ trait AtASolver extends MMUnaryFunc
+ trait AtADenseSolver extends AtASolver
+ trait AtASparseSolver extends AtASolver
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala b/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala
new file mode 100644
index 0000000..6588243
--- /dev/null
+++ b/math-scala/src/main/scala/org/apache/mahout/math/backend/jvm/JvmBackend.scala
@@ -0,0 +1,51 @@
+package org.apache.mahout.math.backend.jvm
+
+import org.apache.mahout.math._
+import scalabindings._
+import RLikeOps._
+import org.apache.mahout.math.backend.Backend
+import org.apache.mahout.math.scalabindings.MMul
+
+import scala.collection.Map
+import scala.reflect._
+
+object JvmBackend extends Backend {
+
+ import org.apache.mahout.math.backend.incore._
+
+ /**
+ * If backend has loaded (lazily) ok and verified its availability/functionality,
+ * this must return `true`.
+ *
+ * @return `true`
+ */
+ override def isAvailable: Boolean = true
+
+ // TODO: In a future release, Refactor MMul optimizations into this object
+ override protected[backend] val solverMap: Map[ClassTag[_], Any] = Map(
+ classTag[MMulSolver] \u2192 MMul
+ // classTag[MMulDenseSolver] \u2192 MMul,
+ // classTag[MMulSparseSolver] \u2192 MMul,
+ // classTag[AtASolver] \u2192 new AtASolver {
+ // override def apply(a: Matrix, r: Option[Matrix]): Matrix = MMul(a.t, a, r)
+ // }// ,
+ // classTag[AtADenseSolver] \u2192 { (a: Matrix, r: Option[Matrix]) \u21d2 MMul(a.t, a, r) },
+ // classTag[AtASparseSolver] \u2192 { (a: Matrix, r: Option[Matrix]) \u21d2 MMul(a.t, a, r) },
+ // classTag[AAtSolver] \u2192 { (a: Matrix, r: Option[Matrix]) \u21d2 MMul(a, a.t, r) },
+ // classTag[AAtDenseSolver] \u2192 { (a: Matrix, r: Option[Matrix]) \u21d2 MMul(a, a.t, r) },
+ // classTag[AAtSparseSolver] \u2192 { (a: Matrix, r: Option[Matrix]) \u21d2 MMul(a, a.t, r) }
+ )
+ validateMap()
+
+ private val mmulSolver = new MMulSolver with MMulDenseSolver with MMulSparseSolver {
+ override def apply(a: Matrix, b: Matrix, r: Option[Matrix]): Matrix = MMul(a, b, r)
+ }
+
+ private val ataSolver = new AtASolver with AtADenseSolver with AtASparseSolver {
+ override def apply(a: Matrix, r: Option[Matrix]): Matrix = MMul(a.t, a, r)
+ }
+
+ private val aatSolver = new AAtSolver {
+ override def apply(a: Matrix, r: Option[Matrix]): Matrix = MMul(a, a.t, r)
+ }
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala
index 2938e5d..f9bda8a 100644
--- a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala
+++ b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/MMul.scala
@@ -22,10 +22,11 @@ import org.apache.mahout.math.flavor.{BackEnum, TraversingStructureEnum}
import org.apache.mahout.math.function.Functions
import RLikeOps._
import org.apache.mahout.logging._
+import org.apache.mahout.math.backend.incore.MMulSolver
import scala.collection.JavaConversions._
-object MMul extends MMBinaryFunc {
+object MMul extends MMulSolver {
private final implicit val log = getLog(MMul.getClass)
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala
index e994e31..3ba6ce0 100644
--- a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala
+++ b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOps.scala
@@ -17,14 +17,24 @@
package org.apache.mahout.math.scalabindings
import org.apache.mahout.math.function.Functions
-import org.apache.mahout.math.{Vector, Matrix}
+import org.apache.mahout.math.{Matrix, Vector}
+
import scala.collection.JavaConversions._
import RLikeOps._
+import org.apache.mahout.math.backend.RootSolverFactory
+import org.apache.mahout.math.scalabindings._
+
class RLikeMatrixOps(m: Matrix) extends MatrixOps(m) {
/** Structure-optimized mmul */
- def %*%(that: Matrix) = MMul(m, that, None)
+
+ implicit var solverOperator: opMMulSolver = _
+
+ // get the solver matching the implicit variable solverOperator
+ def mmulSolver = RootSolverFactory.getOperator
+
+ def %*%(that: Matrix) = mmulSolver(m, that, None)
def :%*%(that:Matrix) = %*%(that)
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala
index 8b1ce65..4115091 100644
--- a/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala
+++ b/math-scala/src/main/scala/org/apache/mahout/math/scalabindings/package.scala
@@ -365,6 +365,9 @@ package object scalabindings {
type VMBinaryFunc = (Vector, Matrix, Option[Matrix]) \u21d2 Matrix
type MDBinaryFunc = (Matrix, Double, Option[Matrix]) \u21d2 Matrix
+ trait opMMulSolver extends MMBinaryFunc {
+
+ }
/////////////////////////////////////
// Miscellaneous in-core utilities
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala b/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala
new file mode 100644
index 0000000..ba6e145
--- /dev/null
+++ b/math-scala/src/test/scala/org/apache/mahout/math/backend/BackendSuite.scala
@@ -0,0 +1,59 @@
+package org.apache.mahout.math.backend
+
+import org.apache.mahout.math.backend.jvm.JvmBackend
+import org.scalatest.{FunSuite, Matchers}
+
+import scala.collection.mutable
+import scala.reflect.{ClassTag, classTag}
+
+class BackendSuite extends FunSuite with Matchers {
+
+ test("GenericBackend") {
+
+ trait MySolverTrait1 { def myMethod1 = Unit }
+
+
+ trait MySolverTrait2
+
+ class MySolverImpl1 extends MySolverTrait1 {
+ }
+
+ class MySolverImpl2 extends MySolverTrait2
+
+ // My dummy backend supporting to trait solvers filled with 2 dummy implementations of these
+ // traits should be able to serve based on their solver traits.
+ val myBackend = new Backend {
+
+ override def isAvailable: Boolean = true
+
+ override val solverMap = new mutable.HashMap[ClassTag[_], Any]()
+
+ solverMap ++= Map(
+ classTag[MySolverTrait1] \u2192 new MySolverImpl1,
+ classTag[MySolverTrait2] \u2192 new MySolverImpl2
+ )
+
+ validateMap()
+ }
+
+ myBackend.getSolver shouldBe None
+
+ val mySolver1 = myBackend.getSolver[MySolverTrait1]
+
+ // This is indeed solver1 trait type:
+ mySolver1.get.myMethod1
+ mySolver1.get.isInstanceOf[MySolverImpl1] shouldBe true
+
+ // Validator should not allow non-subclasses in implementation.
+ an [IllegalArgumentException] mustBe thrownBy {
+ myBackend.solverMap(classTag[MySolverTrait2]) = 0
+ myBackend.validateMap()
+ }
+ }
+
+ test("JvmBackend") {
+ // Just create JVM backend and validate.
+ JvmBackend.validateMap()
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala b/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala
index b44e295..6dc8207 100644
--- a/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala
+++ b/math-scala/src/test/scala/org/apache/mahout/math/scalabindings/RLikeMatrixOpsSuite.scala
@@ -120,6 +120,9 @@ class RLikeMatrixOpsSuite extends FunSuite with MahoutSuite {
System.currentTimeMillis() - ms
}
+
+ // We're not using GPUMMul or OMPMMul in math-scala so dont need to worry about
+ // changing it in this method
def getMmulAvgs(mxA: Matrix, mxB: Matrix, n: Int) = {
var control: Matrix = null
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8e7bfe8..72bd766 100644
--- a/pom.xml
+++ b/pom.xml
@@ -250,6 +250,13 @@
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId>
+ <groupId>${project.groupId}</groupId>
+ <version>${project.version}</version>
+ </dependency>
+
+
<!-- 3rd party -->
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -845,25 +852,27 @@
<module>spark-shell</module>
<module>flink</module>
<module>h2o</module>
+ <module>viennacl</module>
+ <module>viennacl-omp</module>
</modules>
<profiles>
+
<profile>
- <id>hadoop1</id>
- <properties>
- <hadoop.classifier>hadoop1</hadoop.classifier>
- <hadoop.version>1.2.1</hadoop.version>
- </properties>
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <version>${hadoop.version}</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
+ <id>viennacl</id>
+ <modules>
+ <module>viennacl</module>
+ <module>viennacl-omp</module>
+ </modules>
</profile>
+
+ <profile>
+ <id>viennacl-omp</id>
+ <modules>
+ <module>viennacl-omp</module>
+ </modules>
+ </profile>
+
<profile>
<id>hadoop2</id>
<activation>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/runtests.sh
----------------------------------------------------------------------
diff --git a/runtests.sh b/runtests.sh
index daa36ea..642ae54 100755
--- a/runtests.sh
+++ b/runtests.sh
@@ -33,6 +33,7 @@ cd hdfs && mvn test >> $BUILD_OUTPUT 2>&1
cd ../math && mvn test >> $BUILD_OUTPUT 2>&1
cd ../math-scala && mvn test >> $BUILD_OUTPUT 2>&1
cd ../spark && mvn test >> $BUILD_OUTPUT 2>&1
+#cd ../viennacl && mvn test >> $BUILD_OUTPUT 2>&1
#cd ../flink && mvn test >> $BUILD_OUTPUT 2>&1
#cd ../h2o && mvn test >> $BUILD_OUTPUT 2>&1
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark-shell/pom.xml
----------------------------------------------------------------------
diff --git a/spark-shell/pom.xml b/spark-shell/pom.xml
index 732c39b..8b40dc3 100644
--- a/spark-shell/pom.xml
+++ b/spark-shell/pom.xml
@@ -117,6 +117,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ <version>1.2.2</version>
+ </dependency>
+
<!-- 3rd-party -->
<dependency>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
----------------------------------------------------------------------
diff --git a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
index 0c9163a..422af76 100644
--- a/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
+++ b/spark-shell/src/main/scala/org/apache/mahout/sparkbindings/shell/MahoutSparkILoop.scala
@@ -95,7 +95,8 @@ class MahoutSparkILoop extends SparkILoop {
masterUrl = master,
appName = "Mahout Spark Shell",
customJars = jars,
- sparkConf = conf
+ sparkConf = conf,
+ addMahoutJars = true
)
_interp.sparkContext = sdc
@@ -162,7 +163,7 @@ class MahoutSparkILoop extends SparkILoop {
_ __ ___ __ _| |__ ___ _ _| |_
| '_ ` _ \ / _` | '_ \ / _ \| | | | __|
| | | | | | (_| | | | | (_) | |_| | |_
- |_| |_| |_|\__,_|_| |_|\___/ \__,_|\__| version 0.12.2
+ |_| |_| |_|\__,_|_| |_|\___/ \__,_|\__| version 0.13.0
""")
import Properties._
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index f965d38..4e75a62 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -160,6 +160,12 @@
</dependency>
<dependency>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ <version>1.2.2</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-hdfs</artifactId>
<exclusions>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/main/assembly/dependency-reduced.xml
----------------------------------------------------------------------
diff --git a/spark/src/main/assembly/dependency-reduced.xml b/spark/src/main/assembly/dependency-reduced.xml
index a3044da..f981639 100644
--- a/spark/src/main/assembly/dependency-reduced.xml
+++ b/spark/src/main/assembly/dependency-reduced.xml
@@ -42,6 +42,8 @@
<include>com.tdunning:t-digest</include>
<include>org.apache.commons:commons-math3</include>
<include>it.unimi.dsi:fastutil</include>
+ <include>org.apache.mahout:mahout-native-viennacl_2.10</include>
+ <include>org.bytedeco:javacpp</include>
</includes>
</dependencySet>
</dependencySets>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala b/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala
index b57d8ae..ba2adc9 100644
--- a/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala
+++ b/spark/src/main/scala/org/apache/mahout/sparkbindings/blas/ABt.scala
@@ -119,7 +119,7 @@ object ABt {
createCombiner = (t: (Array[K], Array[Int], Matrix)) => {
val (rowKeys, colKeys, block) = t
- val comb = if (block.getFlavor == MatrixFlavor.SPARSELIKE) {
+ val comb = if (!densityAnalysis(block)) {
new SparseMatrix(prodNCol, block.nrow).t
} else {
new DenseMatrix(prodNCol, block.nrow).t
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
index 8fd77e6..7da4729 100644
--- a/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
+++ b/spark/src/main/scala/org/apache/mahout/sparkbindings/package.scala
@@ -64,23 +64,28 @@ package object sparkbindings {
try {
- if (addMahoutJars) {
+ // when not including the artifact, eg. for viennacl , we always need
+ // to load all mahout jars
+ // will need to handle this somehow.
+
+ // if (addMahoutJars) {
// context specific jars
val mcjars = findMahoutContextJars(closeables)
- if (log.isDebugEnabled) {
+// if (log.isDebugEnabled) {
log.debug("Mahout jars:")
mcjars.foreach(j => log.debug(j))
- }
+// }
sparkConf.setJars(jars = mcjars.toSeq ++ customJars)
if (!(customJars.size > 0)) sparkConf.setJars(customJars.toSeq)
- } else {
+// } else {
// In local mode we don't care about jars, do we?
- sparkConf.setJars(customJars.toSeq)
- }
+ // yes adding jars always now since we are not including the artifacts
+ // sparkConf.setJars(customJars.toSeq)
+// }
sparkConf.setAppName(appName).setMaster(masterUrl).set("spark.serializer",
"org.apache.spark.serializer.KryoSerializer").set("spark.kryo.registrator",
@@ -254,7 +259,11 @@ package object sparkbindings {
j.matches(".*mahout-hdfs-\\d.*\\.jar") ||
// no need for mapreduce jar in Spark
// j.matches(".*mahout-mr-\\d.*\\.jar") ||
- j.matches(".*mahout-spark_\\d.*\\.jar")
+ j.matches(".*mahout-spark_\\d.*\\.jar") ||
+ // vcl jars: mahout-native-viennacl_2.10.jar,
+ // mahout-native-viennacl-omp_2.10.jar
+ j.matches(".*mahout-native-viennacl_\\d.*\\\\.jar") ||
+ j.matches(".*mahout-native-viennacl-omp_\\d.*\\.jar")
)
// Tune out "bad" classifiers
.filter(n =>
@@ -264,11 +273,11 @@ package object sparkbindings {
// During maven tests, the maven classpath also creeps in for some reason
!n.matches(".*/.m2/.*")
)
- /* verify jar passed to context
- log.info("\n\n\n")
- mcjars.foreach(j => log.info(j))
- log.info("\n\n\n")
- */
+ /* verify jar passed to context */
+// info("\n\n\n")
+// mcjars.foreach(j => info(j))
+// info("\n\n\n")
+ /**/
mcjars
}
[3/5] mahout git commit: MAHOUT-1885: Inital commit of VCL bindings.
closes apache/mahout#269 closes apache/mahout#261
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/linux-haswell.properties
----------------------------------------------------------------------
diff --git a/viennacl-omp/linux-haswell.properties b/viennacl-omp/linux-haswell.properties
new file mode 100644
index 0000000..52d5cec
--- /dev/null
+++ b/viennacl-omp/linux-haswell.properties
@@ -0,0 +1,28 @@
+platform=linux-haswell
+platform.path.separator=:
+platform.source.suffix=.cpp
+platform.includepath.prefix=-I
+platform.includepath=
+platform.compiler=g++
+platform.compiler.cpp11=-std=c++11
+platform.compiler.default=
+platform.compiler.fastfpu=-msse3 -ffast-math
+platform.compiler.viennacl=-fopenmp -fpermissive
+platform.compiler.nodeprecated=-Wno-deprecated-declarations
+#build for haswell arch with for GCC >= 4.9.0
+platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=haswell -m64 -Wall -O3 -fPIC -shared -s -o\u0020
+#for GCC < 4.9.0 use -march=core-avx2 for haswell arch
+#platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=core-avx2 -m64 -Wall -Ofast -fPIC -shared -s -o\u0020
+#build for native:
+#platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=native -m64 -Wall -Ofast -fPIC -shared -s -o\u0020
+platform.linkpath.prefix=-L
+platform.linkpath.prefix2=-Wl,-rpath,
+platform.linkpath=
+platform.link.prefix=-l
+platform.link.suffix=
+platform.link=
+platform.framework.prefix=-F
+platform.framework.suffix=
+platform.framework=
+platform.library.prefix=lib
+platform.library.suffix=.so
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/linux-x86_64-viennacl.properties
----------------------------------------------------------------------
diff --git a/viennacl-omp/linux-x86_64-viennacl.properties b/viennacl-omp/linux-x86_64-viennacl.properties
new file mode 100644
index 0000000..e5de1fa
--- /dev/null
+++ b/viennacl-omp/linux-x86_64-viennacl.properties
@@ -0,0 +1,24 @@
+platform=linux-x86_64
+platform.path.separator=:
+platform.source.suffix=.cpp
+platform.includepath.prefix=-I
+platform.includepath=
+platform.compiler=g++
+platform.compiler.cpp11=-std=c++11
+platform.compiler.default=
+platform.compiler.fastfpu=-msse3 -ffast-math
+platform.compiler.viennacl=-fopenmp -fpermissive
+platform.compiler.nodeprecated=-Wno-deprecated-declarations
+# platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=x86-64 -m64 -Wall -O3 -fPIC -shared -s -o\u0020
+platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=native -m64 -Wall -Ofast -fPIC -shared -s -o\u0020
+platform.linkpath.prefix=-L
+platform.linkpath.prefix2=-Wl,-rpath,
+platform.linkpath=
+platform.link.prefix=-l
+platform.link.suffix=
+platform.link=
+platform.framework.prefix=-F
+platform.framework.suffix=
+platform.framework=
+platform.library.prefix=lib
+platform.library.suffix=.so
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/pom.xml
----------------------------------------------------------------------
diff --git a/viennacl-omp/pom.xml b/viennacl-omp/pom.xml
new file mode 100644
index 0000000..865ef0d
--- /dev/null
+++ b/viennacl-omp/pom.xml
@@ -0,0 +1,278 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout</artifactId>
+ <version>0.13.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>mahout-native-viennacl-omp_${scala.compat.version}</artifactId>
+
+ <name>Mahout Native VienniaCL OpenMP Bindings</name>
+ <description>Native Structures and interfaces to be used from Mahout math-scala.
+ </description>
+
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <!-- create test jar so other modules can reuse the native test utility classes. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-scala-sources</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-compile</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!--this is what scalatest recommends to do to enable scala tests -->
+
+ <!-- disable surefire -->
+ <!-- disable surefire -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ <!-- enable scalatest -->
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <argLine>-Xmx4g</argLine>
+ </configuration>
+ </plugin>
+
+
+ <!--JavaCPP native build plugin-->
+ <!-- old-style way to get it to compile. -->
+ <!--based on https://github.com/bytedeco/javacpp/wiki/Maven-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <id>javacpp</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <environmentVariables>
+ <LD_LIBRARY_PATH>{project.basedir}/target/classes/org/apache/mahout/javacpp/linalg/linux-x86_64_omp/
+ </LD_LIBRARY_PATH>
+ </environmentVariables>
+ <executable>java</executable>
+ <arguments>
+ <argument>-jar</argument>
+ <argument>${org.bytedeco:javacpp:jar}</argument>
+ <argument>-propertyfile</argument>
+ <argument>linux-x86_64-viennacl.properties</argument>
+ <argument>-classpath</argument>
+ <argument>${project.build.outputDirectory}:${org.scala-lang:scala-library:jar}</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.CompressedMatrix</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.Context</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.MatrixBase</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.DenseRowMatrix</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.DenseColumnMatrix</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.MatMatProdExpression</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.ProdExpression</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.MatrixTransExpression</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.LinalgFunctions</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.Functions</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.VectorBase</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.VCLVector</argument>
+ <argument>org.apache.mahout.viennacl.openmp.javacpp.VecMultExpression</argument>
+ <argument>org.apache.mahout.viennacl.openmp.OMPMMul</argument>
+ <argument>org.apache.mahout.viennacl.openmp.OMPMMul$</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>properties</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mahout-math-scala_${scala.compat.version}</artifactId>
+ </dependency>
+
+ <!-- 3rd-party -->
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+
+ <!-- scala stuff -->
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ </dependency>
+
+ <!-- scala-library for annotations at compile time-->
+ <!--<dependency>-->
+ <!--<groupId>org.scala-lang</groupId>-->
+ <!--<artifactId>scala-library</artifactId>-->
+ <!--<version>${scala.version}</version>-->
+ <!--</dependency>-->
+
+
+ <dependency>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ <version>1.2.4</version>
+ </dependency>
+
+ </dependencies>
+
+
+ <profiles>
+ <profile>
+ <id>mahout-release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-scaladoc</id>
+ <goals>
+ <goal>doc</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>attach-scaladoc-jar</id>
+ <goals>
+ <goal>doc-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>travis</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- Limit memory for unit tests in Travis -->
+ <argLine>-Xmx3g</argLine>
+ <!--<argLine>-Djava.library.path=${project.build.directory}/libs/natives/linux-x86_64:${project.build.directory}/libs/natives/linux:${project.build.directory}/libs/natives/maxosx</argLine>-->
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <!-- Limit memory for integration tests in Travis -->
+ <argLine>-Xmx3g</argLine>
+ <!--<argLine>-Djava.library.path=${project.build.directory}/libs/natives/linux-x86_64:${project.build.directory}/libs/natives/linux:${project.build.directory}/libs/natives/maxosx</argLine>-->
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/runs
----------------------------------------------------------------------
diff --git a/viennacl-omp/runs b/viennacl-omp/runs
new file mode 100644
index 0000000..a152244
--- /dev/null
+++ b/viennacl-omp/runs
@@ -0,0 +1,32 @@
+original
+row-major viennacl::matrix
+ + OCL matrix memory domain after assgn=2
+- dense vcl mmul with fast_copy
+- mmul microbenchmark
+ + Mahout multiplication time: 15699 ms.
+ + ViennaCL/OpenCL multiplication time: 3625 ms.
+ + ompA mem domain:1
+ + ompB mem domain:1
+ + ViennaCL/cpu/OpenMP multiplication time: 2838 ms.
+
+with sys.ArrayCopy, all dense.
+ViennaCLSuite:
+- row-major viennacl::matrix
+ + OCL matrix memory domain after assgn=2
+- dense vcl mmul with fast_copy
+- mmul microbenchmark
+ + Mahout multiplication time: 15407 ms.
+ + ViennaCL/OpenCL multiplication time: 3499 ms.
+ + ompA mem domain:1
+ + ompB mem domain:1
+ + ViennaCL/cpu/OpenMP multiplication time: 2714 ms.
+
+DL latest
+ViennaCLSuite:
+- row-major viennacl::matrix
+ + OCL matrix memory domain after assgn=2
+- dense vcl mmul with fast_copy
+- mmul microbenchmark
+ + Mahout multiplication time: 16076 ms.
+ + ViennaCL/OpenCL multiplication time: 3360 ms.
+ + ViennaCL/cpu/OpenMP multiplication time: 2666 ms.
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/Functions.java
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/Functions.java b/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/Functions.java
new file mode 100644
index 0000000..c2bffe5
--- /dev/null
+++ b/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/Functions.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp;
+
+import org.bytedeco.javacpp.BytePointer;
+import org.bytedeco.javacpp.DoublePointer;
+import org.bytedeco.javacpp.IntPointer;
+import org.bytedeco.javacpp.annotation.*;
+
+import java.nio.DoubleBuffer;
+import java.nio.IntBuffer;
+
+
+@Properties(inherit = Context.class,
+ value = @Platform(
+ library = "jniViennaCL"
+ )
+)
+@Namespace("viennacl")
+public final class Functions {
+
+ private Functions() {
+ }
+
+ // This is (imo) an inconsistency in Vienna cl: almost all operations require MatrixBase, and
+ // fast_copy require type `matrix`, i.e., one of DenseRowMatrix or DenseColumnMatrix.
+ @Name("fast_copy")
+ public static native void fastCopy(DoublePointer srcBegin, DoublePointer srcEnd, @ByRef DenseRowMatrix dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(DoublePointer srcBegin, DoublePointer srcEnd, @ByRef DenseColumnMatrix dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@ByRef DenseRowMatrix src, DoublePointer dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@ByRef DenseColumnMatrix src, DoublePointer dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@Const @ByRef VectorBase dst, @Const @ByRef VCLVector src);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@Const @ByRef VCLVector src, @Const @ByRef VectorBase dst);
+
+
+ @ByVal
+ public static native MatrixTransExpression trans(@ByRef MatrixBase src);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadInt(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ IntPointer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadDouble(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ DoublePointer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadInt(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ IntBuffer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadDouble(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ DoubleBuffer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadBytes(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ BytePointer ptr,
+ boolean async);
+
+
+ static {
+ Context.loadLib();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/LinalgFunctions.java
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/LinalgFunctions.java b/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/LinalgFunctions.java
new file mode 100644
index 0000000..c2a40d9
--- /dev/null
+++ b/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/LinalgFunctions.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp;
+
+import org.apache.mahout.viennacl.openmp.javacpp.*;
+import org.bytedeco.javacpp.annotation.*;
+
+
+@Properties(inherit = Context.class,
+ value = @Platform(
+ library = "jniViennaCL"
+ )
+)
+@Namespace("viennacl::linalg")
+public final class LinalgFunctions {
+
+ private LinalgFunctions() {
+ }
+
+ static {
+ Context.loadLib();
+ }
+
+
+ @ByVal
+ public static native MatMatProdExpression prod(@Const @ByRef MatrixBase a,
+ @Const @ByRef MatrixBase b);
+
+ @ByVal
+ public static native ProdExpression prod(@Const @ByRef CompressedMatrix a,
+ @Const @ByRef CompressedMatrix b);
+
+ @ByVal
+ public static native MatVecProdExpression prod(@Const @ByRef MatrixBase a,
+ @Const @ByRef VectorBase b);
+
+ @ByVal
+ public static native SrMatDnMatProdExpression prod(@Const @ByRef CompressedMatrix spMx,
+ @Const @ByRef MatrixBase dMx);
+ @ByVal
+ @Name("prod")
+ public static native DenseColumnMatrix prodCm(@Const @ByRef MatrixBase a,
+ @Const @ByRef MatrixBase b);
+ @ByVal
+ @Name("prod")
+ public static native DenseRowMatrix prodRm(@Const @ByRef MatrixBase a,
+ @Const @ByRef MatrixBase b);
+
+ @ByVal
+ @Name("prod")
+ public static native DenseRowMatrix prodRm(@Const @ByRef CompressedMatrix spMx,
+ @Const @ByRef MatrixBase dMx);
+
+
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseRowMatrix a,
+// @Const @ByRef DenseRowMatrix b);
+//
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseRowMatrix a,
+// @Const @ByRef DenseColumnMatrix b);
+//
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseColumnMatrix a,
+// @Const @ByRef DenseRowMatrix b);
+//
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseColumnMatrix a,
+// @Const @ByRef DenseColumnMatrix b);
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/MatrixTransExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/MatrixTransExpression.scala b/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/MatrixTransExpression.scala
new file mode 100644
index 0000000..82574b4
--- /dev/null
+++ b/viennacl-omp/src/main/java/org/apache/mahout/viennacl/openmp/javacpp/MatrixTransExpression.scala
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp;
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ include = Array("matrix.hpp"),
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::matrix_base<double>, " +
+ "const viennacl::matrix_base<double>, " +
+ "viennacl::op_trans>"))
+class MatrixTransExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/OMPMMul.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/OMPMMul.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/OMPMMul.scala
new file mode 100644
index 0000000..58a06dd
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/OMPMMul.scala
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.viennacl.openmp
+
+import org.apache.mahout.logging._
+import org.apache.mahout.math
+import org.apache.mahout.math._
+import org.apache.mahout.math.backend.incore.MMulSolver
+import org.apache.mahout.math.flavor.{BackEnum, TraversingStructureEnum}
+import org.apache.mahout.math.function.Functions
+import org.apache.mahout.math.scalabindings.RLikeOps._
+import org.apache.mahout.math.scalabindings._
+import org.apache.mahout.viennacl.openmp.javacpp.Functions._
+import org.apache.mahout.viennacl.openmp.javacpp.LinalgFunctions._
+import org.apache.mahout.viennacl.openmp.javacpp.{CompressedMatrix, Context, DenseRowMatrix}
+
+import scala.collection.JavaConversions._
+
+object OMPMMul extends MMBinaryFunc {
+
+ private final implicit val log = getLog(OMPMMul.getClass)
+
+ override def apply(a: Matrix, b: Matrix, r: Option[Matrix]): Matrix = {
+
+ require(a.ncol == b.nrow, "Incompatible matrix sizes in matrix multiplication.")
+
+ val (af, bf) = (a.getFlavor, b.getFlavor)
+ val backs = (af.getBacking, bf.getBacking)
+ val sd = (af.getStructure, math.scalabindings.densityAnalysis(a), bf.getStructure, densityAnalysis(b))
+
+
+ try {
+
+ val alg: MMulAlg = backs match {
+
+ // Both operands are jvm memory backs.
+ case (BackEnum.JVMMEM, BackEnum.JVMMEM) \u21d2
+
+ sd match {
+
+ // Multiplication cases by a diagonal matrix.
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.COLWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagCW
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.SPARSECOLWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagCW
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.ROWWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagRW
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.SPARSEROWWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagRW
+
+ case (TraversingStructureEnum.COLWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmCWDiag
+ case (TraversingStructureEnum.SPARSECOLWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmCWDiag
+ case (TraversingStructureEnum.ROWWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmRWDiag
+ case (TraversingStructureEnum.SPARSEROWWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmRWDiag
+
+ // Dense-dense cases
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.COLWISE, true) if a eq b.t \u21d2 ompDRWAAt
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.COLWISE, true) if a.t eq b \u21d2 ompDRWAAt
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.COLWISE, true) \u21d2 ompRWCW
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.ROWWISE, true) \u21d2 jvmRWRW
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.COLWISE, true) \u21d2 jvmCWCW
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.ROWWISE, true) if a eq b.t \u21d2 jvmDCWAAt
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.ROWWISE, true) if a.t eq b \u21d2 jvmDCWAAt
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.ROWWISE, true) \u21d2 jvmCWRW
+
+ // Sparse row matrix x sparse row matrix (array of vectors)
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.ROWWISE, false) \u21d2 ompSparseRWRW
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.COLWISE, false) \u21d2 jvmSparseRWCW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.ROWWISE, false) \u21d2 jvmSparseCWRW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.COLWISE, false) \u21d2 jvmSparseCWCW
+
+ // Sparse matrix x sparse matrix (hashtable of vectors)
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2
+ ompSparseRowRWRW
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2
+ jvmSparseRowRWCW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2
+ jvmSparseRowCWRW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2
+ jvmSparseRowCWCW
+
+ // Sparse matrix x non-like
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 ompSparseRowRWRW
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseRowRWCW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 jvmSparseRowCWRW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseCWCW
+ case (TraversingStructureEnum.ROWWISE, _, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2 ompSparseRWRW
+ case (TraversingStructureEnum.ROWWISE, _, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2 jvmSparseRWCW
+ case (TraversingStructureEnum.COLWISE, _, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2 jvmSparseCWRW
+ case (TraversingStructureEnum.COLWISE, _, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2 jvmSparseRowCWCW
+
+ // Everything else including at least one sparse LHS or RHS argument
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 ompSparseRWRW
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseRWCW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 jvmSparseCWRW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseCWCW2flips
+
+ // Sparse methods are only effective if the first argument is sparse, so we need to do a swap.
+ case (_, _, _, false) \u21d2 (a, b, r) \u21d2 apply(b.t, a.t, r.map {
+ _.t
+ }).t
+
+ // Default jvm-jvm case.
+ // for some reason a SrarseRowMatrix DRM %*% SrarseRowMatrix DRM was dumping off to here
+ case _ \u21d2 ompRWCW
+ }
+ }
+
+ alg(a, b, r)
+ } catch {
+ // TODO FASTHACK: just revert to JVM if there is an exception..
+ // eg. java.lang.nullPointerException if more openCL contexts
+ // have been created than number of GPU cards.
+ // better option wuold be to fall back to OpenCl First.
+ case ex: Exception =>
+ println(ex.getMessage + "falling back to JVM MMUL")
+ return MMul(a, b, r)
+ }
+ }
+
+ type MMulAlg = MMBinaryFunc
+
+ @inline
+ private def ompRWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("ompRWCW")
+ //
+ // require(r.forall(mxR \u21d2 mxR.nrow == a.nrow && mxR.ncol == b.ncol))
+ // val (m, n) = (a.nrow, b.ncol)
+ //
+ // val mxR = r.getOrElse(if (densityAnalysis(a)) a.like(m, n) else b.like(m, n))
+ //
+ // for (row \u2190 0 until mxR.nrow; col \u2190 0 until mxR.ncol) {
+ // // this vector-vector should be sort of optimized, right?
+ // mxR(row, col) = a(row, ::) dot b(::, col)
+ // }
+ // mxR
+
+ val hasElementsA = a.zSum() > 0.0
+ val hasElementsB = b.zSum() > 0.0
+
+ // A has a sparse matrix structure of unknown size. We do not want to
+ // simply convert it to a Dense Matrix which may result in an OOM error.
+
+ // If it is empty use JVM MMul, since we can not convert it to a VCL CSR Matrix.
+ if (!hasElementsA) {
+ println("Matrix a has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ // CSR matrices are efficient up to 50% non-zero
+ if (b.getFlavor.isDense) {
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.MAIN_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, oclCtx)
+ val oclB = toVclDenseRM(b, oclCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenMP multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ } else {
+ // Fall back to JVM based MMul if either matrix is sparse and empty
+ if (!hasElementsA || !hasElementsB) {
+ println("Matrix a or b has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ var ms = System.currentTimeMillis()
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, hostClCtx)
+ val oclB = toVclCmpMatrixAlt(b, hostClCtx)
+ val oclC = new CompressedMatrix(prod(oclA, oclB))
+ val mxC = fromVclCompressedMatrix(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenMP multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ }
+ }
+
+
+ @inline
+ private def jvmRWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmRWRW")
+ // A bit hackish: currently, this relies a bit on the fact that like produces RW(?)
+ val bclone = b.like(b.ncol, b.nrow).t
+ for (brow \u2190 b) bclone(brow.index(), ::) := brow
+
+ require(bclone.getFlavor.getStructure == TraversingStructureEnum.COLWISE || bclone.getFlavor.getStructure ==
+ TraversingStructureEnum.SPARSECOLWISE, "COL wise conversion assumption of RHS is wrong, do over this code.")
+
+ ompRWCW(a, bclone, r)
+ }
+
+ private def jvmCWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmCWCW")
+ jvmRWRW(b.t, a.t, r.map(_.t)).t
+ }
+
+ private def jvmCWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmCWRW")
+ // This is a primary contender with Outer Prod sum algo.
+ // Here, we force-reorient both matrices and run RWCW.
+ // A bit hackish: currently, this relies a bit on the fact that clone always produces RW(?)
+ val aclone = a.cloned
+
+ require(aclone.getFlavor.getStructure == TraversingStructureEnum.ROWWISE || aclone.getFlavor.getStructure ==
+ TraversingStructureEnum.SPARSEROWWISE, "Row wise conversion assumption of RHS is wrong, do over this code.")
+
+ jvmRWRW(aclone, b, r)
+ }
+
+ // left is Sparse right is any
+ private def ompSparseRWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("ompSparseRWRW")
+ val mxR = r.getOrElse(b.like(a.nrow, b.ncol))
+
+ // make sure that the matrix is not empty. VCL {{compressed_matrix}}s must
+ // hav nnz > 0
+ // this method is horribly inefficent. however there is a difference between
+ // getNumNonDefaultElements() and getNumNonZeroElements() which we do not always
+ // have access to created MAHOUT-1882 for this
+ val hasElementsA = a.zSum() > 0.0
+ val hasElementsB = b.zSum() > 0.0
+
+ // A has a sparse matrix structure of unknown size. We do not want to
+ // simply convert it to a Dense Matrix which may result in an OOM error.
+ // If it is empty use JVM MMul, since we can not convert it to a VCL CSR Matrix.
+ if (!hasElementsA) {
+ println("Matrix a has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ // CSR matrices are efficient up to 50% non-zero
+ if(b.getFlavor.isDense) {
+ var ms = System.currentTimeMillis()
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, hostClCtx)
+ val oclB = toVclDenseRM(b, hostClCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenMP multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ } else {
+ // Fall back to JVM based MMul if either matrix is sparse and empty
+ if (!hasElementsA || !hasElementsB) {
+ println("Matrix a or b has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ var ms = System.currentTimeMillis()
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, hostClCtx)
+ val oclB = toVclCmpMatrixAlt(b, hostClCtx)
+ val oclC = new CompressedMatrix(prod(oclA, oclB))
+ val mxC = fromVclCompressedMatrix(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenMP multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ }
+
+ }
+
+ //sparse %*% dense
+ private def ompSparseRowRWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("ompSparseRowRWRW")
+ val hasElementsA = a.zSum() > 0
+
+ // A has a sparse matrix structure of unknown size. We do not want to
+ // simply convert it to a Dense Matrix which may result in an OOM error.
+ // If it is empty fall back to JVM MMul, since we can not convert it
+ // to a VCL CSR Matrix.
+ if (!hasElementsA) {
+ println("Matrix a has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ var ms = System.currentTimeMillis()
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, hostClCtx)
+ val oclB = toVclDenseRM(b, hostClCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenMP multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ }
+
+ private def jvmSparseRowCWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ ompSparseRowRWRW(b.t, a.t, r.map(_.t)).t
+
+ private def jvmSparseRowCWCW2flips(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ ompSparseRowRWRW(a cloned, b cloned, r)
+
+ private def jvmSparseRowRWCW(a: Matrix, b: Matrix, r: Option[Matrix]) =
+ ompSparseRowRWRW(a, b cloned, r)
+
+
+ private def jvmSparseRowCWRW(a: Matrix, b: Matrix, r: Option[Matrix]) =
+ ompSparseRowRWRW(a cloned, b, r)
+
+ private def jvmSparseRWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ ompSparseRWRW(a, b.cloned, r)
+
+ private def jvmSparseCWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ ompSparseRWRW(a cloned, b, r)
+
+ private def jvmSparseCWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ ompSparseRWRW(b.t, a.t, r.map(_.t)).t
+
+ private def jvmSparseCWCW2flips(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ ompSparseRWRW(a cloned, b cloned, r)
+
+ private def jvmDiagRW(diagm:Matrix, b:Matrix, r:Option[Matrix] = None):Matrix = {
+ println("jvmDiagRW")
+ val mxR = r.getOrElse(b.like(diagm.nrow, b.ncol))
+
+ for (del \u2190 diagm.diagv.nonZeroes())
+ mxR(del.index, ::).assign(b(del.index, ::), Functions.plusMult(del))
+
+ mxR
+ }
+
+ private def jvmDiagCW(diagm: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmDiagCW")
+ val mxR = r.getOrElse(b.like(diagm.nrow, b.ncol))
+ for (bcol \u2190 b.t) mxR(::, bcol.index()) := bcol * diagm.diagv
+ mxR
+ }
+
+ private def jvmCWDiag(a: Matrix, diagm: Matrix, r: Option[Matrix] = None) =
+ jvmDiagRW(diagm, a.t, r.map {_.t}).t
+
+ private def jvmRWDiag(a: Matrix, diagm: Matrix, r: Option[Matrix] = None) =
+ jvmDiagCW(diagm, a.t, r.map {_.t}).t
+
+
+ /** Dense column-wise AA' */
+ private def jvmDCWAAt(a:Matrix, b:Matrix, r:Option[Matrix] = None) = {
+ // a.t must be equiv. to b. Cloning must rewrite to row-wise.
+ ompDRWAAt(a.cloned,null,r)
+ }
+
+ /** Dense Row-wise AA' */
+ // we probably will not want to use this for the actual release unless A is cached already
+ // but adding for testing purposes.
+ private def ompDRWAAt(a:Matrix, b:Matrix, r:Option[Matrix] = None) = {
+ // a.t must be equiv to b.
+ println("executing on OMP")
+ debug("AAt computation detected; passing off to OMP")
+
+ // Check dimensions if result is supplied.
+ require(r.forall(mxR \u21d2 mxR.nrow == a.nrow && mxR.ncol == a.nrow))
+
+ val mxR = r.getOrElse(a.like(a.nrow, a.nrow))
+
+ var ms = System.currentTimeMillis()
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val oclA = toVclDenseRM(src = a, hostClCtx)
+ val oclAt = new DenseRowMatrix(trans(oclA))
+ val oclC = new DenseRowMatrix(prod(oclA, oclAt))
+
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenMP multiplication time: $ms ms.")
+
+ oclA.close()
+ //oclApr.close()
+ oclAt.close()
+ oclC.close()
+
+ mxC
+
+ }
+
+ private def jvmOuterProdSum(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmOuterProdSum")
+ // This may be already laid out for outer product computation, which may be faster than reorienting
+ // both matrices? need to check.
+ val (m, n) = (a.nrow, b.ncol)
+
+ // Prefer col-wise result iff a is dense and b is sparse. In all other cases default to row-wise.
+ val preferColWiseR = a.getFlavor.isDense && !b.getFlavor.isDense
+
+ val mxR = r.getOrElse {
+ (a.getFlavor.isDense, preferColWiseR) match {
+ case (false, false) \u21d2 b.like(m, n)
+ case (false, true) \u21d2 b.like(n, m).t
+ case (true, false) \u21d2 a.like(m, n)
+ case (true, true) \u21d2 a.like(n, m).t
+ }
+ }
+
+ // Loop outer products
+ if (preferColWiseR) {
+ // this means B is sparse and A is not, so we need to iterate over b values and update R columns with +=
+ // one at a time.
+ for ((acol, brow) \u2190 a.t.zip(b); bel \u2190 brow.nonZeroes) mxR(::, bel.index()) += bel * acol
+ } else {
+ for ((acol, brow) \u2190 a.t.zip(b); ael \u2190 acol.nonZeroes()) mxR(ael.index(), ::) += ael * brow
+ }
+
+ mxR
+ }
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/CompressedMatrix.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/CompressedMatrix.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/CompressedMatrix.scala
new file mode 100644
index 0000000..72f9fad
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/CompressedMatrix.scala
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import java.nio._
+
+import org.bytedeco.javacpp._
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ include = Array("compressed_matrix.hpp"),
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::compressed_matrix<double>"))
+final class CompressedMatrix(defaultCtr: Boolean = true) extends Pointer {
+
+ protected val ptrs = new ArrayBuffer[Pointer]()
+
+ // call this after set or better TODO: yet wrap set() in a public method that will call this
+ def registerPointersForDeallocation(p:Pointer): Unit = {
+ ptrs += p
+ }
+
+ override def deallocate(deallocate: Boolean): Unit = {
+ super.deallocate(deallocate)
+ ptrs.foreach(_.close())
+ }
+
+ if (defaultCtr) allocate()
+
+ def this(nrow: Int, ncol: Int, ctx: Context = new Context) {
+ this(false)
+ allocate(nrow, ncol, ctx)
+ }
+
+ def this(nrow: Int, ncol: Int, nonzeros: Int, ctx: Context = new Context) {
+ this(false)
+ allocate(nrow, ncol, nonzeros, ctx)
+ }
+
+ def this(pe: ProdExpression) {
+ this(false)
+ allocate(pe)
+ }
+
+ @native protected def allocate()
+
+ @native protected def allocate(nrow: Int, ncol: Int, nonzeros: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(nrow: Int, ncol: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(@Const @ByRef pe: ProdExpression)
+
+// @native protected def allocate(db: DoubleBuffer)
+//
+// @native protected def allocate(ib: IntBuffer)
+
+ // Warning: apparently there are differences in bit interpretation between OpenCL and everything
+ // else for unsigned int type. So, for OpenCL backend, rowJumper and colIndices have to be packed
+ // with reference to that cl_uint type that Vienna-CL defines.
+ @native def set(@Cast(Array("const void*")) rowJumper: IntBuffer,
+ @Cast(Array("const void*")) colIndices: IntBuffer,
+ @Const elements: DoubleBuffer,
+ nrow: Int,
+ ncol: Int,
+ nonzeros: Int
+ )
+
+ /** With javacpp pointers. */
+ @native def set(@Cast(Array("const void*")) rowJumper: IntPointer,
+ @Cast(Array("const void*")) colIndices: IntPointer,
+ @Const elements: DoublePointer,
+ nrow: Int,
+ ncol: Int,
+ nonzeros: Int
+ )
+
+ @Name(Array("operator="))
+ @native def :=(@Const @ByRef pe: ProdExpression)
+
+ @native def generate_row_block_information()
+
+ /** getters for the compressed_matrix size */
+ //const vcl_size_t & size1() const { return rows_; }
+ @native def size1: Int
+ //const vcl_size_t & size2() const { return cols_; }
+ @native def size2: Int
+ //const vcl_size_t & nnz() const { return nonzeros_; }
+ @native def nnz: Int
+ //const vcl_size_t & blocks1() const { return row_block_num_; }
+ // @native def blocks1: Int
+
+ /** getters for the compressed_matrix buffers */
+ //const handle_type & handle1() const { return row_buffer_; }
+ @native @Const @ByRef def handle1: MemHandle
+ //const handle_type & handle2() const { return col_buffer_; }
+ @native @Const @ByRef def handle2: MemHandle
+ //const handle_type & handle3() const { return row_blocks_; }
+ @native @Const @ByRef def handle3: MemHandle
+ //const handle_type & handle() const { return elements_; }
+ @native @Const @ByRef def handle: MemHandle
+
+}
+
+object CompressedMatrix {
+ Context.loadLib()
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/Context.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/Context.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/Context.scala
new file mode 100644
index 0000000..ae1b782
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/Context.scala
@@ -0,0 +1,58 @@
+package org.apache.mahout.viennacl.openmp.javacpp
+
+
+import org.bytedeco.javacpp.annotation._
+import org.bytedeco.javacpp.{Loader, Pointer}
+
+/**
+ * This assumes viennacl 1.7.1 is installed, which in ubuntu Xenial defaults to
+ * /usr/include/viennacl, and is installed via
+ * {{{
+ * sudo apt-get install libviennacl-dev
+ * }}}
+ *
+ * @param mtype
+ */
+@Properties(Array(
+ new Platform(
+ includepath = Array("/usr/include/viennacl"),
+ include = Array("matrix.hpp", "compressed_matrix.hpp"),
+ define = Array("VIENNACL_WITH_OPENMP"),
+ compiler = Array("fastfpu","viennacl"),
+ link = Array("OpenCL"),
+ library = "jniViennaCL"
+ )))
+@Namespace("viennacl")
+@Name(Array("context"))
+final class Context(mtype: Int = Context.MEMORY_NOT_INITIALIZED) extends Pointer {
+
+ import Context._
+
+ if (mtype == MEMORY_NOT_INITIALIZED)
+ allocate()
+ else
+ allocate(mtype)
+
+ @native protected def allocate()
+
+ @native protected def allocate(@Cast(Array("viennacl::memory_types")) mtype: Int)
+
+ @Name(Array("memory_type"))
+ @Cast(Array("int"))
+ @native def memoryType: Int
+
+}
+
+object Context {
+
+ def loadLib() = Loader.load(classOf[Context])
+
+ loadLib()
+
+ /* Memory types. Ported from VCL header files. */
+ val MEMORY_NOT_INITIALIZED = 0
+ val MAIN_MEMORY = 1
+ val OPENCL_MEMORY = 2
+ val CUDA_MEMORY = 3
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseColumnMatrix.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseColumnMatrix.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseColumnMatrix.scala
new file mode 100644
index 0000000..eeab17b
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseColumnMatrix.scala
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.DoublePointer
+import org.bytedeco.javacpp.annotation._
+
+/**
+ * ViennaCL dense matrix, column-major. This is an exact duplication of [[DenseRowMatrix]], and
+ * is only different in the materialized C++ template name. Unfortunately I so far have not figured
+ * out how to handle it with.
+ *
+ * Also, the [[Platform.library]] does not get inherited for some reason, and we really want to
+ * collect all class mappings in the same one libjni.so, so we have to repeat this `library` defi-
+ * nition in every mapped class in this package. (One .so per package convention).
+ */
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform (
+ include=Array("matrix.hpp"),
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::matrix<double,viennacl::column_major>"))
+final class DenseColumnMatrix(initDefault:Boolean = true) extends MatrixBase {
+
+ def this(nrow: Int, ncol: Int, ctx: Context = new Context()) {
+ this(false)
+ allocate(nrow, ncol, ctx)
+ }
+
+ def this(data: DoublePointer, nrow: Int, ncol: Int, ctx: Context = new Context(Context.MAIN_MEMORY)) {
+ this(false)
+ allocate(data, ctx.memoryType, nrow, ncol)
+ // We save it to deallocate it ad deallocation time.
+ ptrs += data
+ }
+
+ def this(me: MatMatProdExpression) {
+ this(false)
+ allocate(me)
+ }
+
+ def this(me: MatrixTransExpression) {
+ this(false)
+ allocate(me)
+ }
+
+
+ if (initDefault) allocate()
+
+ @native protected def allocate()
+
+ @native protected def allocate(nrow: Int, ncol: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(data: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))
+ memType: Int,
+ nrow: Int,
+ ncol: Int
+ )
+
+ @native protected def allocate(@Const @ByRef me: MatMatProdExpression)
+
+ @native protected def allocate(@Const @ByRef me: MatrixTransExpression)
+
+}
+
+object DenseColumnMatrix {
+ Context.loadLib()
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseRowMatrix.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseRowMatrix.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseRowMatrix.scala
new file mode 100644
index 0000000..3281465
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/DenseRowMatrix.scala
@@ -0,0 +1,69 @@
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.DoublePointer
+import org.bytedeco.javacpp.annotation._
+
+/**
+ * ViennaCL dense matrix, row-major.
+ */
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL"
+ )))
+@Name(Array("viennacl::matrix<double,viennacl::row_major>"))
+class DenseRowMatrix(initDefault: Boolean = true) extends MatrixBase {
+
+ def this(nrow: Int, ncol: Int, ctx: Context = new Context()) {
+ this(false)
+ allocate(nrow, ncol, ctx)
+ }
+
+ def this(data: DoublePointer, nrow: Int, ncol: Int, ctx: Context = new Context(Context.MAIN_MEMORY)) {
+ this(false)
+ allocate(data, ctx.memoryType, nrow, ncol)
+ // We save it to deallocate it ad deallocation time.
+ ptrs += data
+ }
+
+ def this(me: MatMatProdExpression) {
+ this(false)
+ allocate(me)
+ }
+
+ def this(me: MatrixTransExpression) {
+ this(false)
+ allocate(me)
+ }
+
+ // TODO: getting compilation errors here
+ def this(sd: SrMatDnMatProdExpression) {
+ this(false)
+ allocate(sd)
+ }
+
+ if (initDefault) allocate()
+
+ @native protected def allocate()
+
+ @native protected def allocate(nrow: Int, ncol: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(data: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))
+ memType: Int,
+ nrow: Int,
+ ncol: Int
+ )
+
+ @native protected def allocate(@Const @ByRef me: MatMatProdExpression)
+
+ @native protected def allocate(@Const @ByRef me: MatrixTransExpression)
+
+ // TODO: Compilation errors here
+ @native protected def allocate(@Const @ByRef me: SrMatDnMatProdExpression)
+
+}
+
+
+object DenseRowMatrix {
+ Context.loadLib()
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatMatProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatMatProdExpression.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatMatProdExpression.scala
new file mode 100644
index 0000000..c15bbd9
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatMatProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::matrix_base<double>, " +
+ "const viennacl::matrix_base<double>, " +
+ "viennacl::op_mat_mat_prod>"))
+class MatMatProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatVecProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatVecProdExpression.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatVecProdExpression.scala
new file mode 100644
index 0000000..4435232
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatVecProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("vector_expression<const viennacl::matrix_base<double>, " +
+ "const viennacl::vector_base<double>, " +
+ "viennacl::op_prod>"))
+class MatVecProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatrixBase.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatrixBase.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatrixBase.scala
new file mode 100644
index 0000000..00823b6
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MatrixBase.scala
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL"
+ )))
+@Name(Array("viennacl::matrix_base<double>"))
+class MatrixBase extends Pointer {
+
+ protected val ptrs = new ArrayBuffer[Pointer]()
+
+ override def deallocate(deallocate: Boolean): Unit = {
+ super.deallocate(deallocate)
+ ptrs.foreach(_.close())
+ }
+
+ @Name(Array("operator="))
+ @native def :=(@Const @ByRef src: DenseRowMatrix)
+
+ @Name(Array("operator="))
+ @native def :=(@Const @ByRef src: DenseColumnMatrix)
+
+ @Name(Array("size1"))
+ @native
+ def nrow: Int
+
+ @Name(Array("size2"))
+ @native
+ def ncol: Int
+
+ @Name(Array("row_major"))
+ @native
+ def isRowMajor: Boolean
+
+ @Name(Array("internal_size1"))
+ @native
+ def internalnrow: Int
+
+ @Name(Array("internal_size2"))
+ @native
+ def internalncol: Int
+
+ @Name(Array("memory_domain"))
+ @native
+ def memoryDomain: Int
+
+ @Name(Array("switch_memory_context"))
+ @native
+ def switchMemoryContext(@ByRef ctx: Context)
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MemHandle.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MemHandle.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MemHandle.scala
new file mode 100644
index 0000000..938a262
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/MemHandle.scala
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation._
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl::backend")
+@Name(Array("mem_handle"))
+class MemHandle extends Pointer {
+
+ allocate()
+
+ @native def allocate()
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/ProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/ProdExpression.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/ProdExpression.scala
new file mode 100644
index 0000000..315a03c
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/ProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::compressed_matrix<double>, " +
+ "const viennacl::compressed_matrix<double>, " +
+ "viennacl::op_prod>"))
+class ProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/SrMatDnMatProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/SrMatDnMatProdExpression.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/SrMatDnMatProdExpression.scala
new file mode 100644
index 0000000..e9c7bac
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/SrMatDnMatProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::compressed_matrix<double>, " +
+ "const viennacl::matrix_base<double>, " +
+ "viennacl::op_prod>"))
+class SrMatDnMatProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VCLVector.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VCLVector.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VCLVector.scala
new file mode 100644
index 0000000..987c947
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VCLVector.scala
@@ -0,0 +1,115 @@
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp._
+import org.bytedeco.javacpp.annotation._
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::vector<double>"))
+final class VCLVector(defaultCtr: Boolean = true) extends VectorBase {
+
+ if (defaultCtr) allocate()
+
+ def this(){
+ this(false)
+ allocate()
+ }
+
+ def this(i: Int){
+ this(false)
+ allocate(i)
+ }
+
+ def this(size: Int, ctx: Context = new Context(Context.MAIN_MEMORY)) {
+ this(false)
+ allocate(size, ctx)
+ }
+
+ def this(@Const @ByRef ve: VecMultExpression) {
+ this(false)
+ allocate(ve)
+ }
+
+ def this(@Const @ByRef vmp: MatVecProdExpression) {
+ this(false)
+ allocate(vmp)
+ }
+
+// conflicting with the next signature as MemHandle is a pointer and so is a DoublePointer..
+// leave out for now.
+//
+// def this(h: MemHandle , vec_size: Int, vec_start: Int = 0, vec_stride: Int = 1) {
+// this(false)
+// allocate(h, vec_size, vec_start, vec_stride)
+// }
+
+ def this(ptr_to_mem: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))mem_type : Int,
+ vec_size: Int,
+ start: Int = 0,
+ stride: Int = 1) {
+
+ this(false)
+ allocate(ptr_to_mem, mem_type, vec_size, start, stride)
+ ptrs += ptr_to_mem
+ }
+
+ def this(@Const @ByRef vc: VCLVector) {
+ this(false)
+ allocate(vc)
+ }
+ def this(@Const @ByRef vb: VectorBase) {
+ this(false)
+ allocate(vb)
+ }
+
+ @native protected def allocate()
+
+ @native protected def allocate(size: Int)
+
+ @native protected def allocate(size: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(@Const @ByRef ve: VecMultExpression)
+
+ @native protected def allocate(@Const @ByRef ve: MatVecProdExpression)
+
+ @native protected def allocate(@Const @ByRef vb: VCLVector)
+
+ @native protected def allocate(@Const @ByRef vb: VectorBase)
+
+
+// @native protected def allocate(h: MemHandle , vec_size: Int,
+// vec_start: Int,
+// vec_stride: Int)
+
+ @native protected def allocate(ptr_to_mem: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))mem_type : Int,
+ vec_size: Int,
+ start: Int,
+ stride: Int)
+
+ @Name(Array("viennacl::vector<double>::self_type"))
+ def selfType:VectorBase = this.asInstanceOf[VectorBase]
+
+
+ @native def switch_memory_context(@ByVal context: Context): Unit
+
+// Swaps the handles of two vectors by swapping the OpenCL handles only, no data copy.
+// @native def fast_swap(@ByVal other: VCLVector): VectorBase
+
+// add this operator in for tests many more can be added
+// @Name(Array("operator*"))
+// @native @ByPtr def *(i: Int): VectorMultExpression
+
+
+
+}
+
+object VCLVector {
+ Context.loadLib()
+}
+
+
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VecMultExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VecMultExpression.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VecMultExpression.scala
new file mode 100644
index 0000000..7562de5
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VecMultExpression.scala
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("vector_expression<const viennacl::vector_base<double>," +
+ "const double, viennacl::op_mult >"))
+class VecMultExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VectorBase.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VectorBase.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VectorBase.scala
new file mode 100644
index 0000000..8efd377
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/javacpp/VectorBase.scala
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.openmp.javacpp
+
+import org.bytedeco.javacpp._
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::vector_base<double>"))
+class VectorBase extends Pointer {
+
+ protected val ptrs = new ArrayBuffer[Pointer]()
+
+ override def deallocate(deallocate: Boolean): Unit = {
+ super.deallocate(deallocate)
+ ptrs.foreach(_.close())
+ }
+
+ // size of the vec elements
+ @native @Const def size(): Int
+
+ // size of the vec elements + padding
+ @native @Const def internal_size(): Int
+
+ // handle to the vec element buffer
+ @native @Const @ByRef def handle: MemHandle
+
+// // add this operator in for tests many more can be added
+// @Name(Array("operator* "))
+// @native def *(i: Int): VectorMultExpression
+
+
+}
+
+
[4/5] mahout git commit: MAHOUT-1885: Inital commit of VCL bindings.
closes apache/mahout#269 closes apache/mahout#261
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
index 628d981..fc84577 100644
--- a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriverSuite.scala
@@ -1,832 +1,832 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.drivers
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{Path, FileSystem}
-import org.apache.mahout.math.indexeddataset.{BiDictionary, IndexedDataset}
-import org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark
-import org.scalatest.{ConfigMap, FunSuite}
-import org.apache.mahout.sparkbindings._
-import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
-import org.apache.mahout.math.drm._
-import org.apache.mahout.math.scalabindings._
-
-import scala.collection.immutable.HashMap
-
-//todo: take out, only for temp tests
-
-import org.apache.mahout.math.scalabindings._
-import RLikeOps._
-import org.apache.mahout.math.drm._
-import RLikeDrmOps._
-import scala.collection.JavaConversions._
-
-
-class ItemSimilarityDriverSuite extends FunSuite with DistributedSparkSuite {
-
- /*
- final val matrixLLRCoocAtAControl = dense(
- (0.0, 0.6331745808516107, 0.0, 0.0, 0.0),
- (0.6331745808516107, 0.0, 0.0, 0.0, 0.0),
- (0.0, 0.0, 0.0, 0.6331745808516107, 0.0),
- (0.0, 0.0, 0.6331745808516107, 0.0, 0.0),
- (0.0, 0.0, 0.0, 0.0, 0.0))
-
- final val matrixLLRCoocBtAControl = dense(
- (1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 0.0),
- (0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.0),
- (0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.0),
- (1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 0.0),
- (0.0, 0.0, 0.6795961471815897, 0.0, 4.498681156950466))
- */
-
-
- final val SelfSimilairtyLines = Iterable(
- "galaxy\tnexus:1.7260924347106847",
- "ipad\tiphone:1.7260924347106847",
- "nexus\tgalaxy:1.7260924347106847",
- "iphone\tipad:1.7260924347106847",
- "surface")
-
- val CrossSimilarityLines = Iterable(
- "iphone\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
- "ipad\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
- "nexus\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
- "galaxy\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
- "surface\tsurface:4.498681156950466 nexus:0.6795961471815897")
-
- // todo: a better test would be to sort each vector by itemID and compare rows, tokens misses some error cases
- final val SelfSimilairtyTokens = tokenize(Iterable(
- "galaxy\tnexus:1.7260924347106847",
- "ipad\tiphone:1.7260924347106847",
- "nexus\tgalaxy:1.7260924347106847",
- "iphone\tipad:1.7260924347106847",
- "surface"))
-
- val CrossSimilarityTokens = tokenize(Iterable(
- "iphone\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
- "ipad\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
- "nexus\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
- "galaxy\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
- "surface\tsurface:4.498681156950466 nexus:0.6795961471815897"))
-
- /*
- //Clustered Spark and HDFS, not a good everyday build test
- ItemSimilarityDriver.main(Array(
- "--input", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/cf-data.txt",
- "--output", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/similarityMatrices/",
- "--master", "spark://occam4:7077",
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1"))
- */
- // local multi-threaded Spark with HDFS using large dataset
- // not a good build test.
- /*
- ItemSimilarityDriver.main(Array(
- "--input", "hdfs://occam4:54310/user/pat/xrsj/ratings_data.txt",
- "--output", "hdfs://occam4:54310/user/pat/xrsj/similarityMatrices/",
- "--master", "local[4]",
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1"))
- */
-
- test("ItemSimilarityDriver, non-full-spec CSV") {
-
- val InFile = TmpDir + "in-file.csv/" //using part files, not single file
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1,purchase,iphone",
- "u1,purchase,ipad",
- "u2,purchase,nexus",
- "u2,purchase,galaxy",
- "u3,purchase,surface",
- "u4,purchase,iphone",
- "u4,purchase,galaxy",
- "u1,view,iphone",
- "u1,view,ipad",
- "u1,view,nexus",
- "u1,view,galaxy",
- "u2,view,iphone",
- "u2,view,ipad",
- "u2,view,nexus",
- "u2,view,galaxy",
- "u3,view,surface",
- "u3,view,nexus",
- "u4,view,iphone",
- "u4,view,ipad",
- "u4,view,galaxy")
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(InFile)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1",
- "--writeAllDatasets"))
-
- // todo: these comparisons rely on a sort producing the same lines, which could possibly
- // fail since the sort is on value and these can be the same for all items in a vector
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
- }
-
-
-
- test("ItemSimilarityDriver TSV ") {
-
- val InFile = TmpDir + "in-file.tsv/"
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1\tpurchase\tiphone",
- "u1\tpurchase\tipad",
- "u2\tpurchase\tnexus",
- "u2\tpurchase\tgalaxy",
- "u3\tpurchase\tsurface",
- "u4\tpurchase\tiphone",
- "u4\tpurchase\tgalaxy",
- "u1\tview\tiphone",
- "u1\tview\tipad",
- "u1\tview\tnexus",
- "u1\tview\tgalaxy",
- "u2\tview\tiphone",
- "u2\tview\tipad",
- "u2\tview\tnexus",
- "u2\tview\tgalaxy",
- "u3\tview\tsurface",
- "u3\tview\tnexus",
- "u4\tview\tiphone",
- "u4\tview\tipad",
- "u4\tview\tgalaxy")
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(InFile)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", "[,\t]",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1"))
-
- // todo: a better test would be to get sorted vectors and compare rows instead of tokens, this might miss
- // some error cases
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
-
- }
-
- test("ItemSimilarityDriver log-ish files") {
-
- val InFile = TmpDir + "in-file.log/"
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "2014-06-23 14:46:53.115\tu1\tpurchase\trandom text\tiphone",
- "2014-06-23 14:46:53.115\tu1\tpurchase\trandom text\tipad",
- "2014-06-23 14:46:53.115\tu2\tpurchase\trandom text\tnexus",
- "2014-06-23 14:46:53.115\tu2\tpurchase\trandom text\tgalaxy",
- "2014-06-23 14:46:53.115\tu3\tpurchase\trandom text\tsurface",
- "2014-06-23 14:46:53.115\tu4\tpurchase\trandom text\tiphone",
- "2014-06-23 14:46:53.115\tu4\tpurchase\trandom text\tgalaxy",
- "2014-06-23 14:46:53.115\tu1\tview\trandom text\tiphone",
- "2014-06-23 14:46:53.115\tu1\tview\trandom text\tipad",
- "2014-06-23 14:46:53.115\tu1\tview\trandom text\tnexus",
- "2014-06-23 14:46:53.115\tu1\tview\trandom text\tgalaxy",
- "2014-06-23 14:46:53.115\tu2\tview\trandom text\tiphone",
- "2014-06-23 14:46:53.115\tu2\tview\trandom text\tipad",
- "2014-06-23 14:46:53.115\tu2\tview\trandom text\tnexus",
- "2014-06-23 14:46:53.115\tu2\tview\trandom text\tgalaxy",
- "2014-06-23 14:46:53.115\tu3\tview\trandom text\tsurface",
- "2014-06-23 14:46:53.115\tu3\tview\trandom text\tnexus",
- "2014-06-23 14:46:53.115\tu4\tview\trandom text\tiphone",
- "2014-06-23 14:46:53.115\tu4\tview\trandom text\tipad",
- "2014-06-23 14:46:53.115\tu4\tview\trandom text\tgalaxy")
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(InFile)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", "\t",
- "--itemIDColumn", "4",
- "--rowIDColumn", "1",
- "--filterColumn", "2"))
-
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
-
- }
-
- test("ItemSimilarityDriver legacy supported file format") {
-
- val InDir = TmpDir + "in-dir/"
- val InFilename = "in-file.tsv"
- val InPath = InDir + InFilename
-
- val OutPath = TmpDir + "similarity-matrices"
-
- val lines = Array(
- "0,0,1",
- "0,1,1",
- "1,2,1",
- "1,3,1",
- "2,4,1",
- "3,0,1",
- "3,3,1")
-
- val Answer = tokenize(Iterable(
- "0\t1:1.7260924347106847",
- "3\t2:1.7260924347106847",
- "1\t0:1.7260924347106847",
- "4",
- "2\t3:1.7260924347106847"))
-
- // this creates one part-0000 file in the directory
- mahoutCtx.parallelize(lines).coalesce(1, shuffle = true).saveAsTextFile(InDir)
-
- // to change from using part files to a single .tsv file we'll need to use HDFS
- val fs = FileSystem.get(new Configuration())
- //rename part-00000 to something.tsv
- fs.rename(new Path(InDir + "part-00000"), new Path(InPath))
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InPath,
- "--output", OutPath,
- "--master", masterUrl))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs Answer
-
- }
-
- test("ItemSimilarityDriver write search engine output") {
-
- val InDir = TmpDir + "in-dir/"
- val InFilename = "in-file.tsv"
- val InPath = InDir + InFilename
-
- val OutPath = TmpDir + "similarity-matrices"
-
- val lines = Array(
- "0,0,1",
- "0,1,1",
- "1,2,1",
- "1,3,1",
- "2,4,1",
- "3,0,1",
- "3,3,1")
-
- val Answer = tokenize(Iterable(
- "0\t1",
- "3\t2",
- "1\t0",
- "4",
- "2\t3"))
-
- // this creates one part-0000 file in the directory
- mahoutCtx.parallelize(lines).coalesce(1, shuffle = true).saveAsTextFile(InDir)
-
- // to change from using part files to a single .tsv file we'll need to use HDFS
- val fs = FileSystem.get(new Configuration())
- //rename part-00000 to something.tsv
- fs.rename(new Path(InDir + "part-00000"), new Path(InPath))
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InPath,
- "--output", OutPath,
- "--master", masterUrl,
- "--omitStrength"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs Answer
-
- }
-
- test("ItemSimilarityDriver recursive file discovery using filename patterns") {
- //directory structure using the following
- // tmp/data/m1.tsv
- // tmp/data/more-data/another-dir/m2.tsv
- val M1Lines = Array(
- "u1\tpurchase\tiphone",
- "u1\tpurchase\tipad",
- "u2\tpurchase\tnexus",
- "u2\tpurchase\tgalaxy",
- "u3\tpurchase\tsurface",
- "u4\tpurchase\tiphone",
- "u4\tpurchase\tgalaxy")
-
- val M2Lines = Array(
- "u1\tview\tiphone",
- "u1\tview\tipad",
- "u1\tview\tnexus",
- "u1\tview\tgalaxy",
- "u2\tview\tiphone",
- "u2\tview\tipad",
- "u2\tview\tnexus",
- "u2\tview\tgalaxy",
- "u3\tview\tsurface",
- "u3\tview\tnexus",
- "u4\tview\tiphone",
- "u4\tview\tipad",
- "u4\tview\tgalaxy")
-
- val InFilenameM1 = "m1.tsv"
- val InDirM1 = TmpDir + "data/"
- val InPathM1 = InDirM1 + InFilenameM1
- val InFilenameM2 = "m2.tsv"
- val InDirM2 = TmpDir + "data/more-data/another-dir/"
- val InPathM2 = InDirM2 + InFilenameM2
-
- val InPathStart = TmpDir + "data/"
- val OutPath = TmpDir + "similarity-matrices"
-
- // this creates one part-0000 file in the directory
- mahoutCtx.parallelize(M1Lines).coalesce(1, shuffle = true).saveAsTextFile(InDirM1)
-
- // to change from using part files to a single .tsv file we'll need to use HDFS
- val fs = FileSystem.get(new Configuration())
- //rename part-00000 to something.tsv
- fs.rename(new Path(InDirM1 + "part-00000"), new Path(InPathM1))
-
- // this creates one part-0000 file in the directory
- mahoutCtx.parallelize(M2Lines).coalesce(1, shuffle = true).saveAsTextFile(InDirM2)
-
- // to change from using part files to a single .tsv file we'll need to use HDFS
- //rename part-00000 to tmp/some-location/something.tsv
- fs.rename(new Path(InDirM2 + "part-00000"), new Path(InPathM2))
-
- // local multi-threaded Spark with default FS, suitable for build tests but need better location for data
-
- ItemSimilarityDriver.main(Array(
- "--input", InPathStart,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", "\t",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1",
- "--filenamePattern", "m..tsv",
- "--recursive"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
-
- }
-
- test("ItemSimilarityDriver, two input paths") {
-
- val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
- val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1,purchase,iphone",
- "u1,purchase,ipad",
- "u2,purchase,nexus",
- "u2,purchase,galaxy",
- "u3,purchase,surface",
- "u4,purchase,iphone",
- "u4,purchase,galaxy",
- "u1,view,iphone",
- "u1,view,ipad",
- "u1,view,nexus",
- "u1,view,galaxy",
- "u2,view,iphone",
- "u2,view,ipad",
- "u2,view,nexus",
- "u2,view,galaxy",
- "u3,view,surface",
- "u3,view,nexus",
- "u4,view,iphone",
- "u4,view,ipad",
- "u4,view,galaxy")
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
- val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile1,
- "--input2", InFile2,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
-
- }
-
- test("ItemSimilarityDriver, two inputs of different dimensions") {
-
- val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
- val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1,purchase,iphone",
- "u1,purchase,ipad",
- "u2,purchase,nexus",
- "u2,purchase,galaxy",
- // remove one user so A'B will be of different dimensions
- // ItemSimilarityDriver should create one unified user dictionary and so account for this
- // discrepancy as a blank row: "u3,purchase,surface",
- "u4,purchase,iphone",
- "u4,purchase,galaxy",
- "u1,view,iphone",
- "u1,view,ipad",
- "u1,view,nexus",
- "u1,view,galaxy",
- "u2,view,iphone",
- "u2,view,ipad",
- "u2,view,nexus",
- "u2,view,galaxy",
- "u3,view,surface",
- "u3,view,nexus",
- "u4,view,iphone",
- "u4,view,ipad",
- "u4,view,galaxy")
-
- val UnequalDimensionsSelfSimilarity = tokenize(Iterable(
- "ipad\tiphone:1.7260924347106847",
- "iphone\tipad:1.7260924347106847",
- "nexus\tgalaxy:1.7260924347106847",
- "galaxy\tnexus:1.7260924347106847"))
-
- //only surface purchase was removed so no cross-similarity for surface
- val UnequalDimensionsCrossSimilarity = tokenize(Iterable(
- "galaxy\tgalaxy:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 nexus:1.7260924347106847",
- "iphone\tgalaxy:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 nexus:1.7260924347106847",
- "ipad\tgalaxy:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 nexus:0.6795961471815897",
- "nexus\tiphone:0.6795961471815897 ipad:0.6795961471815897 nexus:0.6795961471815897 galaxy:0.6795961471815897"))
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
- val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile1,
- "--input2", InFile2,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs UnequalDimensionsSelfSimilarity
- tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarity
-
- }
-
- test("ItemSimilarityDriver cross similarity two separate items spaces") {
- /* cross-similarity with category views, same user space
- phones tablets mobile_acc soap
- u1 0 1 1 0
- u2 1 1 1 0
- u3 0 0 1 0
- u4 1 1 0 1
- */
- val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
- val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1,purchase,iphone",
- "u1,purchase,ipad",
- "u2,purchase,nexus",
- "u2,purchase,galaxy",
- "u3,purchase,surface",
- "u4,purchase,iphone",
- "u4,purchase,galaxy",
- "u1,view,phones",
- "u1,view,mobile_acc",
- "u2,view,phones",
- "u2,view,tablets",
- "u2,view,mobile_acc",
- "u3,view,mobile_acc",
- "u4,view,phones",
- "u4,view,tablets",
- "u4,view,soap")
-
- val UnequalDimensionsCrossSimilarityLines = tokenize(Iterable(
- "iphone\tmobile_acc:1.7260924347106847 soap:1.7260924347106847 phones:1.7260924347106847",
- "surface\tmobile_acc:0.6795961471815897",
- "nexus\ttablets:1.7260924347106847 mobile_acc:0.6795961471815897 phones:0.6795961471815897",
- "galaxy\ttablets:5.545177444479561 soap:1.7260924347106847 phones:1.7260924347106847 " +
- "mobile_acc:1.7260924347106847",
- "ipad\tmobile_acc:0.6795961471815897 phones:0.6795961471815897"))
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
- val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile1,
- "--input2", InFile2,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1",
- "--writeAllDatasets"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarityLines
-
- }
-
- test("A.t %*% B after changing row cardinality of A") {
- // todo: move to math tests but this is Spark specific
-
- val a = dense(
- (1.0, 1.0))
-
- val b = dense(
- (1.0, 1.0),
- (1.0, 1.0),
- (1.0, 1.0))
-
- val inCoreABiggertBAnswer = dense(
- (1.0, 1.0),
- (1.0, 1.0))
-
- val drmA = drmParallelize(m = a, numPartitions = 2)
- val drmB = drmParallelize(m = b, numPartitions = 2)
-
- // modified to return a new CheckpointedDrm so maintains immutability but still only increases the row cardinality
- // by returning new CheckpointedDrmSpark[K](rdd, n, ncol, _cacheStorageLevel ) Hack for now.
- val drmABigger = drmWrap[Int](drmA.rdd, 3, 2)
-
-
- val ABiggertB = drmABigger.t %*% drmB
- val inCoreABiggertB = ABiggertB.collect
-
- assert(inCoreABiggertB === inCoreABiggertBAnswer)
-
- val bp = 0
- }
-
- test("Changing row cardinality of an IndexedDataset") {
-
- val a = dense(
- (1.0, 1.0))
-
- val drmA = drmParallelize(m = a, numPartitions = 2)
- val emptyIDs = new BiDictionary(new HashMap[String, Int]())
- val indexedDatasetA = new IndexedDatasetSpark(drmA, emptyIDs, emptyIDs)
- val biggerIDSA = indexedDatasetA.newRowCardinality(5)
-
- assert(biggerIDSA.matrix.nrow == 5)
-
- }
-
- test("ItemSimilarityDriver cross similarity two separate items spaces, missing rows in B") {
- /* cross-similarity with category views, same user space
- phones tablets mobile_acc soap
- u1 0 1 1 0
- u2 1 1 1 0
-removed ==> u3 0 0 1 0
- u4 1 1 0 1
- */
- val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
- val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1,purchase,iphone",
- "u1,purchase,ipad",
- "u2,purchase,nexus",
- "u2,purchase,galaxy",
- "u3,purchase,surface",
- "u4,purchase,iphone",
- "u4,purchase,galaxy",
- "u1,view,phones",
- "u1,view,mobile_acc",
- "u2,view,phones",
- "u2,view,tablets",
- "u2,view,mobile_acc",
- //"u3,view,mobile_acc",// if this line is removed the cross-cooccurrence should work
- "u4,view,phones",
- "u4,view,tablets",
- "u4,view,soap")
-
- val UnequalDimensionsCrossSimilarityLines = tokenize(Iterable(
- "galaxy\ttablets:5.545177444479561 soap:1.7260924347106847 phones:1.7260924347106847",
- "ipad\tmobile_acc:1.7260924347106847 phones:0.6795961471815897",
- "surface",
- "nexus\tmobile_acc:1.7260924347106847 tablets:1.7260924347106847 phones:0.6795961471815897",
- "iphone\tsoap:1.7260924347106847 phones:1.7260924347106847"))
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
- val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile1,
- "--input2", InFile2,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1",
- "--writeAllDatasets"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
- tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarityLines
- }
-
- test("ItemSimilarityDriver cross similarity two separate items spaces, adding rows in B") {
- /* cross-similarity with category views, same user space
- phones tablets mobile_acc soap
- u1 0 1 1 0
- u2 1 1 1 0
-removed ==> u3 0 0 1 0
- u4 1 1 0 1
- */
- val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
- val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
- val OutPath = TmpDir + "similarity-matrices/"
-
- val lines = Array(
- "u1,purchase,iphone",
- "u1,purchase,ipad",
- "u2,purchase,nexus",
- "u2,purchase,galaxy",
- "u3,purchase,surface",
- "u4,purchase,iphone",
- "u4,purchase,galaxy",
- "u1,view,phones",
- "u1,view,mobile_acc",
- "u2,view,phones",
- "u2,view,tablets",
- "u2,view,mobile_acc",
- "u3,view,mobile_acc",// if this line is removed the cross-cooccurrence should work
- "u4,view,phones",
- "u4,view,tablets",
- "u4,view,soap",
- "u5,view,soap")
-
- val UnequalDimensionsSimilarityTokens = List(
- "galaxy",
- "nexus:2.231435513142097",
- "iphone:0.13844293808390518",
- "nexus",
- "galaxy:2.231435513142097",
- "ipad",
- "iphone:2.231435513142097",
- "surface",
- "iphone",
- "ipad:2.231435513142097",
- "galaxy:0.13844293808390518")
-
- val UnequalDimensionsCrossSimilarityLines = List(
- "galaxy",
- "tablets:6.730116670092563",
- "phones:2.9110316603236868",
- "soap:0.13844293808390518",
- "mobile_acc:0.13844293808390518",
- "nexus",
- "tablets:2.231435513142097",
- "mobile_acc:1.184939225613002",
- "phones:1.184939225613002",
- "ipad", "mobile_acc:1.184939225613002",
- "phones:1.184939225613002",
- "surface",
- "mobile_acc:1.184939225613002",
- "iphone",
- "phones:2.9110316603236868",
- "soap:0.13844293808390518",
- "tablets:0.13844293808390518",
- "mobile_acc:0.13844293808390518")
-
- // this will create multiple part-xxxxx files in the InFile dir but other tests will
- // take account of one actual file
- val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
- val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
-
- // local multi-threaded Spark with default HDFS
- ItemSimilarityDriver.main(Array(
- "--input", InFile1,
- "--input2", InFile2,
- "--output", OutPath,
- "--master", masterUrl,
- "--filter1", "purchase",
- "--filter2", "view",
- "--inDelim", ",",
- "--itemIDColumn", "2",
- "--rowIDColumn", "0",
- "--filterColumn", "1",
- "--writeAllDatasets"))
-
- val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
- val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
- tokenize(similarityLines) should contain theSameElementsAs UnequalDimensionsSimilarityTokens
- tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarityLines
- }
-
- // convert into an Iterable of tokens for 'should contain theSameElementsAs Iterable'
- def tokenize(a: Iterable[String]): Iterable[String] = {
- var r: Iterable[String] = Iterable()
- a.foreach { l =>
- l.split("\t").foreach { s =>
- r = r ++ s.split("[\t ]")
- }
- }
- r
- }
-
- override protected def beforeAll(configMap: ConfigMap) {
- super.beforeAll(configMap)
- ItemSimilarityDriver.useContext(mahoutCtx)
- }
-
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.mahout.drivers
+//
+//import org.apache.hadoop.conf.Configuration
+//import org.apache.hadoop.fs.{Path, FileSystem}
+//import org.apache.mahout.math.indexeddataset.{BiDictionary, IndexedDataset}
+//import org.apache.mahout.sparkbindings.indexeddataset.IndexedDatasetSpark
+//import org.scalatest.{ConfigMap, FunSuite}
+//import org.apache.mahout.sparkbindings._
+//import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
+//import org.apache.mahout.math.drm._
+//import org.apache.mahout.math.scalabindings._
+//
+//import scala.collection.immutable.HashMap
+//
+////todo: take out, only for temp tests
+//
+//import org.apache.mahout.math.scalabindings._
+//import RLikeOps._
+//import org.apache.mahout.math.drm._
+//import RLikeDrmOps._
+//import scala.collection.JavaConversions._
+//
+//
+//class ItemSimilarityDriverSuite extends FunSuite with DistributedSparkSuite {
+//
+// /*
+// final val matrixLLRCoocAtAControl = dense(
+// (0.0, 0.6331745808516107, 0.0, 0.0, 0.0),
+// (0.6331745808516107, 0.0, 0.0, 0.0, 0.0),
+// (0.0, 0.0, 0.0, 0.6331745808516107, 0.0),
+// (0.0, 0.0, 0.6331745808516107, 0.0, 0.0),
+// (0.0, 0.0, 0.0, 0.0, 0.0))
+//
+// final val matrixLLRCoocBtAControl = dense(
+// (1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 0.0),
+// (0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.0),
+// (0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.6795961471815897, 0.0),
+// (1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 1.7260924347106847, 0.0),
+// (0.0, 0.0, 0.6795961471815897, 0.0, 4.498681156950466))
+// */
+//
+//
+// final val SelfSimilairtyLines = Iterable(
+// "galaxy\tnexus:1.7260924347106847",
+// "ipad\tiphone:1.7260924347106847",
+// "nexus\tgalaxy:1.7260924347106847",
+// "iphone\tipad:1.7260924347106847",
+// "surface")
+//
+// val CrossSimilarityLines = Iterable(
+// "iphone\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
+// "ipad\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
+// "nexus\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
+// "galaxy\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
+// "surface\tsurface:4.498681156950466 nexus:0.6795961471815897")
+//
+// // todo: a better test would be to sort each vector by itemID and compare rows, tokens misses some error cases
+// final val SelfSimilairtyTokens = tokenize(Iterable(
+// "galaxy\tnexus:1.7260924347106847",
+// "ipad\tiphone:1.7260924347106847",
+// "nexus\tgalaxy:1.7260924347106847",
+// "iphone\tipad:1.7260924347106847",
+// "surface"))
+//
+// val CrossSimilarityTokens = tokenize(Iterable(
+// "iphone\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
+// "ipad\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
+// "nexus\tnexus:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 galaxy:0.6795961471815897",
+// "galaxy\tnexus:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 galaxy:1.7260924347106847",
+// "surface\tsurface:4.498681156950466 nexus:0.6795961471815897"))
+//
+// /*
+// //Clustered Spark and HDFS, not a good everyday build test
+// ItemSimilarityDriver.main(Array(
+// "--input", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/cf-data.txt",
+// "--output", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/similarityMatrices/",
+// "--master", "spark://occam4:7077",
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1"))
+// */
+// // local multi-threaded Spark with HDFS using large dataset
+// // not a good build test.
+// /*
+// ItemSimilarityDriver.main(Array(
+// "--input", "hdfs://occam4:54310/user/pat/xrsj/ratings_data.txt",
+// "--output", "hdfs://occam4:54310/user/pat/xrsj/similarityMatrices/",
+// "--master", "local[4]",
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1"))
+// */
+//
+// test("ItemSimilarityDriver, non-full-spec CSV") {
+//
+// val InFile = TmpDir + "in-file.csv/" //using part files, not single file
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1,purchase,iphone",
+// "u1,purchase,ipad",
+// "u2,purchase,nexus",
+// "u2,purchase,galaxy",
+// "u3,purchase,surface",
+// "u4,purchase,iphone",
+// "u4,purchase,galaxy",
+// "u1,view,iphone",
+// "u1,view,ipad",
+// "u1,view,nexus",
+// "u1,view,galaxy",
+// "u2,view,iphone",
+// "u2,view,ipad",
+// "u2,view,nexus",
+// "u2,view,galaxy",
+// "u3,view,surface",
+// "u3,view,nexus",
+// "u4,view,iphone",
+// "u4,view,ipad",
+// "u4,view,galaxy")
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(InFile)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1",
+// "--writeAllDatasets"))
+//
+// // todo: these comparisons rely on a sort producing the same lines, which could possibly
+// // fail since the sort is on value and these can be the same for all items in a vector
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
+// }
+//
+//
+//
+// test("ItemSimilarityDriver TSV ") {
+//
+// val InFile = TmpDir + "in-file.tsv/"
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1\tpurchase\tiphone",
+// "u1\tpurchase\tipad",
+// "u2\tpurchase\tnexus",
+// "u2\tpurchase\tgalaxy",
+// "u3\tpurchase\tsurface",
+// "u4\tpurchase\tiphone",
+// "u4\tpurchase\tgalaxy",
+// "u1\tview\tiphone",
+// "u1\tview\tipad",
+// "u1\tview\tnexus",
+// "u1\tview\tgalaxy",
+// "u2\tview\tiphone",
+// "u2\tview\tipad",
+// "u2\tview\tnexus",
+// "u2\tview\tgalaxy",
+// "u3\tview\tsurface",
+// "u3\tview\tnexus",
+// "u4\tview\tiphone",
+// "u4\tview\tipad",
+// "u4\tview\tgalaxy")
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(InFile)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", "[,\t]",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1"))
+//
+// // todo: a better test would be to get sorted vectors and compare rows instead of tokens, this might miss
+// // some error cases
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
+//
+// }
+//
+// test("ItemSimilarityDriver log-ish files") {
+//
+// val InFile = TmpDir + "in-file.log/"
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "2014-06-23 14:46:53.115\tu1\tpurchase\trandom text\tiphone",
+// "2014-06-23 14:46:53.115\tu1\tpurchase\trandom text\tipad",
+// "2014-06-23 14:46:53.115\tu2\tpurchase\trandom text\tnexus",
+// "2014-06-23 14:46:53.115\tu2\tpurchase\trandom text\tgalaxy",
+// "2014-06-23 14:46:53.115\tu3\tpurchase\trandom text\tsurface",
+// "2014-06-23 14:46:53.115\tu4\tpurchase\trandom text\tiphone",
+// "2014-06-23 14:46:53.115\tu4\tpurchase\trandom text\tgalaxy",
+// "2014-06-23 14:46:53.115\tu1\tview\trandom text\tiphone",
+// "2014-06-23 14:46:53.115\tu1\tview\trandom text\tipad",
+// "2014-06-23 14:46:53.115\tu1\tview\trandom text\tnexus",
+// "2014-06-23 14:46:53.115\tu1\tview\trandom text\tgalaxy",
+// "2014-06-23 14:46:53.115\tu2\tview\trandom text\tiphone",
+// "2014-06-23 14:46:53.115\tu2\tview\trandom text\tipad",
+// "2014-06-23 14:46:53.115\tu2\tview\trandom text\tnexus",
+// "2014-06-23 14:46:53.115\tu2\tview\trandom text\tgalaxy",
+// "2014-06-23 14:46:53.115\tu3\tview\trandom text\tsurface",
+// "2014-06-23 14:46:53.115\tu3\tview\trandom text\tnexus",
+// "2014-06-23 14:46:53.115\tu4\tview\trandom text\tiphone",
+// "2014-06-23 14:46:53.115\tu4\tview\trandom text\tipad",
+// "2014-06-23 14:46:53.115\tu4\tview\trandom text\tgalaxy")
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(InFile)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", "\t",
+// "--itemIDColumn", "4",
+// "--rowIDColumn", "1",
+// "--filterColumn", "2"))
+//
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
+//
+// }
+//
+// test("ItemSimilarityDriver legacy supported file format") {
+//
+// val InDir = TmpDir + "in-dir/"
+// val InFilename = "in-file.tsv"
+// val InPath = InDir + InFilename
+//
+// val OutPath = TmpDir + "similarity-matrices"
+//
+// val lines = Array(
+// "0,0,1",
+// "0,1,1",
+// "1,2,1",
+// "1,3,1",
+// "2,4,1",
+// "3,0,1",
+// "3,3,1")
+//
+// val Answer = tokenize(Iterable(
+// "0\t1:1.7260924347106847",
+// "3\t2:1.7260924347106847",
+// "1\t0:1.7260924347106847",
+// "4",
+// "2\t3:1.7260924347106847"))
+//
+// // this creates one part-0000 file in the directory
+// mahoutCtx.parallelize(lines).coalesce(1, shuffle = true).saveAsTextFile(InDir)
+//
+// // to change from using part files to a single .tsv file we'll need to use HDFS
+// val fs = FileSystem.get(new Configuration())
+// //rename part-00000 to something.tsv
+// fs.rename(new Path(InDir + "part-00000"), new Path(InPath))
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InPath,
+// "--output", OutPath,
+// "--master", masterUrl))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs Answer
+//
+// }
+//
+// test("ItemSimilarityDriver write search engine output") {
+//
+// val InDir = TmpDir + "in-dir/"
+// val InFilename = "in-file.tsv"
+// val InPath = InDir + InFilename
+//
+// val OutPath = TmpDir + "similarity-matrices"
+//
+// val lines = Array(
+// "0,0,1",
+// "0,1,1",
+// "1,2,1",
+// "1,3,1",
+// "2,4,1",
+// "3,0,1",
+// "3,3,1")
+//
+// val Answer = tokenize(Iterable(
+// "0\t1",
+// "3\t2",
+// "1\t0",
+// "4",
+// "2\t3"))
+//
+// // this creates one part-0000 file in the directory
+// mahoutCtx.parallelize(lines).coalesce(1, shuffle = true).saveAsTextFile(InDir)
+//
+// // to change from using part files to a single .tsv file we'll need to use HDFS
+// val fs = FileSystem.get(new Configuration())
+// //rename part-00000 to something.tsv
+// fs.rename(new Path(InDir + "part-00000"), new Path(InPath))
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InPath,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--omitStrength"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs Answer
+//
+// }
+//
+// test("ItemSimilarityDriver recursive file discovery using filename patterns") {
+// //directory structure using the following
+// // tmp/data/m1.tsv
+// // tmp/data/more-data/another-dir/m2.tsv
+// val M1Lines = Array(
+// "u1\tpurchase\tiphone",
+// "u1\tpurchase\tipad",
+// "u2\tpurchase\tnexus",
+// "u2\tpurchase\tgalaxy",
+// "u3\tpurchase\tsurface",
+// "u4\tpurchase\tiphone",
+// "u4\tpurchase\tgalaxy")
+//
+// val M2Lines = Array(
+// "u1\tview\tiphone",
+// "u1\tview\tipad",
+// "u1\tview\tnexus",
+// "u1\tview\tgalaxy",
+// "u2\tview\tiphone",
+// "u2\tview\tipad",
+// "u2\tview\tnexus",
+// "u2\tview\tgalaxy",
+// "u3\tview\tsurface",
+// "u3\tview\tnexus",
+// "u4\tview\tiphone",
+// "u4\tview\tipad",
+// "u4\tview\tgalaxy")
+//
+// val InFilenameM1 = "m1.tsv"
+// val InDirM1 = TmpDir + "data/"
+// val InPathM1 = InDirM1 + InFilenameM1
+// val InFilenameM2 = "m2.tsv"
+// val InDirM2 = TmpDir + "data/more-data/another-dir/"
+// val InPathM2 = InDirM2 + InFilenameM2
+//
+// val InPathStart = TmpDir + "data/"
+// val OutPath = TmpDir + "similarity-matrices"
+//
+// // this creates one part-0000 file in the directory
+// mahoutCtx.parallelize(M1Lines).coalesce(1, shuffle = true).saveAsTextFile(InDirM1)
+//
+// // to change from using part files to a single .tsv file we'll need to use HDFS
+// val fs = FileSystem.get(new Configuration())
+// //rename part-00000 to something.tsv
+// fs.rename(new Path(InDirM1 + "part-00000"), new Path(InPathM1))
+//
+// // this creates one part-0000 file in the directory
+// mahoutCtx.parallelize(M2Lines).coalesce(1, shuffle = true).saveAsTextFile(InDirM2)
+//
+// // to change from using part files to a single .tsv file we'll need to use HDFS
+// //rename part-00000 to tmp/some-location/something.tsv
+// fs.rename(new Path(InDirM2 + "part-00000"), new Path(InPathM2))
+//
+// // local multi-threaded Spark with default FS, suitable for build tests but need better location for data
+//
+// ItemSimilarityDriver.main(Array(
+// "--input", InPathStart,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", "\t",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1",
+// "--filenamePattern", "m..tsv",
+// "--recursive"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
+//
+// }
+//
+// test("ItemSimilarityDriver, two input paths") {
+//
+// val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
+// val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1,purchase,iphone",
+// "u1,purchase,ipad",
+// "u2,purchase,nexus",
+// "u2,purchase,galaxy",
+// "u3,purchase,surface",
+// "u4,purchase,iphone",
+// "u4,purchase,galaxy",
+// "u1,view,iphone",
+// "u1,view,ipad",
+// "u1,view,nexus",
+// "u1,view,galaxy",
+// "u2,view,iphone",
+// "u2,view,ipad",
+// "u2,view,nexus",
+// "u2,view,galaxy",
+// "u3,view,surface",
+// "u3,view,nexus",
+// "u4,view,iphone",
+// "u4,view,ipad",
+// "u4,view,galaxy")
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
+// val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile1,
+// "--input2", InFile2,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(crossSimilarityLines) should contain theSameElementsAs CrossSimilarityTokens
+//
+// }
+//
+// test("ItemSimilarityDriver, two inputs of different dimensions") {
+//
+// val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
+// val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1,purchase,iphone",
+// "u1,purchase,ipad",
+// "u2,purchase,nexus",
+// "u2,purchase,galaxy",
+// // remove one user so A'B will be of different dimensions
+// // ItemSimilarityDriver should create one unified user dictionary and so account for this
+// // discrepancy as a blank row: "u3,purchase,surface",
+// "u4,purchase,iphone",
+// "u4,purchase,galaxy",
+// "u1,view,iphone",
+// "u1,view,ipad",
+// "u1,view,nexus",
+// "u1,view,galaxy",
+// "u2,view,iphone",
+// "u2,view,ipad",
+// "u2,view,nexus",
+// "u2,view,galaxy",
+// "u3,view,surface",
+// "u3,view,nexus",
+// "u4,view,iphone",
+// "u4,view,ipad",
+// "u4,view,galaxy")
+//
+// val UnequalDimensionsSelfSimilarity = tokenize(Iterable(
+// "ipad\tiphone:1.7260924347106847",
+// "iphone\tipad:1.7260924347106847",
+// "nexus\tgalaxy:1.7260924347106847",
+// "galaxy\tnexus:1.7260924347106847"))
+//
+// //only surface purchase was removed so no cross-similarity for surface
+// val UnequalDimensionsCrossSimilarity = tokenize(Iterable(
+// "galaxy\tgalaxy:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 nexus:1.7260924347106847",
+// "iphone\tgalaxy:1.7260924347106847 iphone:1.7260924347106847 ipad:1.7260924347106847 nexus:1.7260924347106847",
+// "ipad\tgalaxy:0.6795961471815897 iphone:0.6795961471815897 ipad:0.6795961471815897 nexus:0.6795961471815897",
+// "nexus\tiphone:0.6795961471815897 ipad:0.6795961471815897 nexus:0.6795961471815897 galaxy:0.6795961471815897"))
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
+// val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile1,
+// "--input2", InFile2,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs UnequalDimensionsSelfSimilarity
+// tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarity
+//
+// }
+//
+// test("ItemSimilarityDriver cross similarity two separate items spaces") {
+// /* cross-similarity with category views, same user space
+// phones tablets mobile_acc soap
+// u1 0 1 1 0
+// u2 1 1 1 0
+// u3 0 0 1 0
+// u4 1 1 0 1
+// */
+// val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
+// val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1,purchase,iphone",
+// "u1,purchase,ipad",
+// "u2,purchase,nexus",
+// "u2,purchase,galaxy",
+// "u3,purchase,surface",
+// "u4,purchase,iphone",
+// "u4,purchase,galaxy",
+// "u1,view,phones",
+// "u1,view,mobile_acc",
+// "u2,view,phones",
+// "u2,view,tablets",
+// "u2,view,mobile_acc",
+// "u3,view,mobile_acc",
+// "u4,view,phones",
+// "u4,view,tablets",
+// "u4,view,soap")
+//
+// val UnequalDimensionsCrossSimilarityLines = tokenize(Iterable(
+// "iphone\tmobile_acc:1.7260924347106847 soap:1.7260924347106847 phones:1.7260924347106847",
+// "surface\tmobile_acc:0.6795961471815897",
+// "nexus\ttablets:1.7260924347106847 mobile_acc:0.6795961471815897 phones:0.6795961471815897",
+// "galaxy\ttablets:5.545177444479561 soap:1.7260924347106847 phones:1.7260924347106847 " +
+// "mobile_acc:1.7260924347106847",
+// "ipad\tmobile_acc:0.6795961471815897 phones:0.6795961471815897"))
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
+// val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile1,
+// "--input2", InFile2,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1",
+// "--writeAllDatasets"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarityLines
+//
+// }
+//
+// test("A.t %*% B after changing row cardinality of A") {
+// // todo: move to math tests but this is Spark specific
+//
+// val a = dense(
+// (1.0, 1.0))
+//
+// val b = dense(
+// (1.0, 1.0),
+// (1.0, 1.0),
+// (1.0, 1.0))
+//
+// val inCoreABiggertBAnswer = dense(
+// (1.0, 1.0),
+// (1.0, 1.0))
+//
+// val drmA = drmParallelize(m = a, numPartitions = 2)
+// val drmB = drmParallelize(m = b, numPartitions = 2)
+//
+// // modified to return a new CheckpointedDrm so maintains immutability but still only increases the row cardinality
+// // by returning new CheckpointedDrmSpark[K](rdd, n, ncol, _cacheStorageLevel ) Hack for now.
+// val drmABigger = drmWrap[Int](drmA.rdd, 3, 2)
+//
+//
+// val ABiggertB = drmABigger.t %*% drmB
+// val inCoreABiggertB = ABiggertB.collect
+//
+// assert(inCoreABiggertB === inCoreABiggertBAnswer)
+//
+// val bp = 0
+// }
+//
+// test("Changing row cardinality of an IndexedDataset") {
+//
+// val a = dense(
+// (1.0, 1.0))
+//
+// val drmA = drmParallelize(m = a, numPartitions = 2)
+// val emptyIDs = new BiDictionary(new HashMap[String, Int]())
+// val indexedDatasetA = new IndexedDatasetSpark(drmA, emptyIDs, emptyIDs)
+// val biggerIDSA = indexedDatasetA.newRowCardinality(5)
+//
+// assert(biggerIDSA.matrix.nrow == 5)
+//
+// }
+//
+// test("ItemSimilarityDriver cross similarity two separate items spaces, missing rows in B") {
+// /* cross-similarity with category views, same user space
+// phones tablets mobile_acc soap
+// u1 0 1 1 0
+// u2 1 1 1 0
+//removed ==> u3 0 0 1 0
+// u4 1 1 0 1
+// */
+// val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
+// val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1,purchase,iphone",
+// "u1,purchase,ipad",
+// "u2,purchase,nexus",
+// "u2,purchase,galaxy",
+// "u3,purchase,surface",
+// "u4,purchase,iphone",
+// "u4,purchase,galaxy",
+// "u1,view,phones",
+// "u1,view,mobile_acc",
+// "u2,view,phones",
+// "u2,view,tablets",
+// "u2,view,mobile_acc",
+// //"u3,view,mobile_acc",// if this line is removed the cross-cooccurrence should work
+// "u4,view,phones",
+// "u4,view,tablets",
+// "u4,view,soap")
+//
+// val UnequalDimensionsCrossSimilarityLines = tokenize(Iterable(
+// "galaxy\ttablets:5.545177444479561 soap:1.7260924347106847 phones:1.7260924347106847",
+// "ipad\tmobile_acc:1.7260924347106847 phones:0.6795961471815897",
+// "surface",
+// "nexus\tmobile_acc:1.7260924347106847 tablets:1.7260924347106847 phones:0.6795961471815897",
+// "iphone\tsoap:1.7260924347106847 phones:1.7260924347106847"))
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
+// val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile1,
+// "--input2", InFile2,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1",
+// "--writeAllDatasets"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs SelfSimilairtyTokens
+// tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarityLines
+// }
+//
+// test("ItemSimilarityDriver cross similarity two separate items spaces, adding rows in B") {
+// /* cross-similarity with category views, same user space
+// phones tablets mobile_acc soap
+// u1 0 1 1 0
+// u2 1 1 1 0
+//removed ==> u3 0 0 1 0
+// u4 1 1 0 1
+// */
+// val InFile1 = TmpDir + "in-file1.csv/" //using part files, not single file
+// val InFile2 = TmpDir + "in-file2.csv/" //using part files, not single file
+// val OutPath = TmpDir + "similarity-matrices/"
+//
+// val lines = Array(
+// "u1,purchase,iphone",
+// "u1,purchase,ipad",
+// "u2,purchase,nexus",
+// "u2,purchase,galaxy",
+// "u3,purchase,surface",
+// "u4,purchase,iphone",
+// "u4,purchase,galaxy",
+// "u1,view,phones",
+// "u1,view,mobile_acc",
+// "u2,view,phones",
+// "u2,view,tablets",
+// "u2,view,mobile_acc",
+// "u3,view,mobile_acc",// if this line is removed the cross-cooccurrence should work
+// "u4,view,phones",
+// "u4,view,tablets",
+// "u4,view,soap",
+// "u5,view,soap")
+//
+// val UnequalDimensionsSimilarityTokens = List(
+// "galaxy",
+// "nexus:2.231435513142097",
+// "iphone:0.13844293808390518",
+// "nexus",
+// "galaxy:2.231435513142097",
+// "ipad",
+// "iphone:2.231435513142097",
+// "surface",
+// "iphone",
+// "ipad:2.231435513142097",
+// "galaxy:0.13844293808390518")
+//
+// val UnequalDimensionsCrossSimilarityLines = List(
+// "galaxy",
+// "tablets:6.730116670092563",
+// "phones:2.9110316603236868",
+// "soap:0.13844293808390518",
+// "mobile_acc:0.13844293808390518",
+// "nexus",
+// "tablets:2.231435513142097",
+// "mobile_acc:1.184939225613002",
+// "phones:1.184939225613002",
+// "ipad", "mobile_acc:1.184939225613002",
+// "phones:1.184939225613002",
+// "surface",
+// "mobile_acc:1.184939225613002",
+// "iphone",
+// "phones:2.9110316603236868",
+// "soap:0.13844293808390518",
+// "tablets:0.13844293808390518",
+// "mobile_acc:0.13844293808390518")
+//
+// // this will create multiple part-xxxxx files in the InFile dir but other tests will
+// // take account of one actual file
+// val linesRdd1 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile1)
+// val linesRdd2 = mahoutCtx.parallelize(lines).saveAsTextFile(InFile2)
+//
+// // local multi-threaded Spark with default HDFS
+// ItemSimilarityDriver.main(Array(
+// "--input", InFile1,
+// "--input2", InFile2,
+// "--output", OutPath,
+// "--master", masterUrl,
+// "--filter1", "purchase",
+// "--filter2", "view",
+// "--inDelim", ",",
+// "--itemIDColumn", "2",
+// "--rowIDColumn", "0",
+// "--filterColumn", "1",
+// "--writeAllDatasets"))
+//
+// val similarityLines = mahoutCtx.textFile(OutPath + "/similarity-matrix/").collect.toIterable
+// val crossSimilarityLines = mahoutCtx.textFile(OutPath + "/cross-similarity-matrix/").collect.toIterable
+// tokenize(similarityLines) should contain theSameElementsAs UnequalDimensionsSimilarityTokens
+// tokenize(crossSimilarityLines) should contain theSameElementsAs UnequalDimensionsCrossSimilarityLines
+// }
+//
+// // convert into an Iterable of tokens for 'should contain theSameElementsAs Iterable'
+// def tokenize(a: Iterable[String]): Iterable[String] = {
+// var r: Iterable[String] = Iterable()
+// a.foreach { l =>
+// l.split("\t").foreach { s =>
+// r = r ++ s.split("[\t ]")
+// }
+// }
+// r
+// }
+//
+// override protected def beforeAll(configMap: ConfigMap) {
+// super.beforeAll(configMap)
+// ItemSimilarityDriver.useContext(mahoutCtx)
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/test/scala/org/apache/mahout/drivers/RowSimilarityDriverSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/drivers/RowSimilarityDriverSuite.scala b/spark/src/test/scala/org/apache/mahout/drivers/RowSimilarityDriverSuite.scala
index eccddb1..e6f917c 100644
--- a/spark/src/test/scala/org/apache/mahout/drivers/RowSimilarityDriverSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/drivers/RowSimilarityDriverSuite.scala
@@ -1,139 +1,139 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.drivers
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.fs.{FileSystem, Path}
-import org.apache.mahout.math.drm.RLikeDrmOps._
-import org.apache.mahout.math.drm._
-import org.apache.mahout.math.scalabindings.RLikeOps._
-import org.apache.mahout.math.scalabindings._
-import org.apache.mahout.sparkbindings._
-import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
-import org.scalatest.{ConfigMap, FunSuite}
-
-
-class RowSimilarityDriverSuite extends FunSuite with DistributedSparkSuite {
-
- val TextDocs = Array(
- "doc1\tNow is the time for all good people to come to aid of their party",
- "doc2\tNow is the time for all good people to come to aid of their country",
- "doc3\tNow is the time for all good people to come to aid of their hood",
- "doc4\tNow is the time for all good people to come to aid of their friends",
- "doc5\tNow is the time for all good people to come to aid of their looser brother",
- "doc6\tThe quick brown fox jumped over the lazy dog",
- "doc7\tThe quick brown fox jumped over the lazy boy",
- "doc8\tThe quick brown fox jumped over the lazy cat",
- "doc9\tThe quick brown fox jumped over the lazy wolverine",
- "doc10\tThe quick brown fox jumped over the lazy cantelope")// yes that's spelled correctly.
-
- test("RowSimilarityDriver text docs no strengths") {
-
- val firstFiveSimDocsTokens = tokenize(Iterable(
- "doc1\tdoc3 doc2 doc4 doc5"))
-
- val lastFiveSimDocsTokens = tokenize(Iterable(
- "doc6\tdoc8 doc10 doc7 doc9"))
-
- val inDir = TmpDir + "in-dir/"
- val inFilename = "in-file.tsv"
- val inPath = inDir + inFilename
-
- val outPath = TmpDir + "similarity-matrices/"
-
-
- // this creates one part-0000 file in the directory
- mahoutCtx.parallelize(TextDocs).coalesce(1, shuffle=true).saveAsTextFile(inDir)
-
- // to change from using part files to a single .tsv file we'll need to use HDFS
- val fs = FileSystem.get(new Configuration())
- //rename part-00000 to something.tsv
- fs.rename(new Path(inDir + "part-00000"), new Path(inPath))
-
- // local multi-threaded Spark with default HDFS
- RowSimilarityDriver.main(Array(
- "--input", inPath,
- "--output", outPath,
- "--omitStrength",
- "--maxSimilaritiesPerRow", "4", // would get all docs similar if we didn't limit them
- "--master", masterUrl))
-
- val simLines = mahoutCtx.textFile(outPath).collect
- simLines.foreach { line =>
- val lineTokens = line.split("[\t ]")
- if (lineTokens.contains("doc1") ) // docs are two flavors so if only 4 similarities it will effectively classify
- lineTokens should contain theSameElementsAs firstFiveSimDocsTokens
- else
- lineTokens should contain theSameElementsAs lastFiveSimDocsTokens
- }
-
- }
-
- test("RowSimilarityDriver text docs") {
-
- val simDocsTokens = tokenize(Iterable(
- "doc1\tdoc3:27.87301122947484 doc2:27.87301122947484 doc4:27.87301122947484 doc5:23.42278065550721",
- "doc2\tdoc4:27.87301122947484 doc3:27.87301122947484 doc1:27.87301122947484 doc5:23.42278065550721",
- "doc3\tdoc4:27.87301122947484 doc2:27.87301122947484 doc1:27.87301122947484 doc5:23.42278065550721",
- "doc4\tdoc3:27.87301122947484 doc2:27.87301122947484 doc1:27.87301122947484 doc5:23.42278065550721",
- "doc5\tdoc4:23.42278065550721 doc2:23.42278065550721 doc3:23.42278065550721 doc1:23.42278065550721",
- "doc6\tdoc8:22.936393049704463 doc10:22.936393049704463 doc7:22.936393049704463 doc9:22.936393049704463",
- "doc7\tdoc6:22.936393049704463 doc8:22.936393049704463 doc10:22.936393049704463 doc9:22.936393049704463",
- "doc8\tdoc6:22.936393049704463 doc10:22.936393049704463 doc7:22.936393049704463 doc9:22.936393049704463",
- "doc9\tdoc6:22.936393049704463 doc8:22.936393049704463 doc10:22.936393049704463 doc7:22.936393049704463",
- "doc10\tdoc6:22.936393049704463 doc8:22.936393049704463 doc7:22.936393049704463 doc9:22.936393049704463"))
-
- val inDir = TmpDir + "in-dir/"
- val inFilename = "in-file.tsv"
- val inPath = inDir + inFilename
-
- val outPath = TmpDir + "similarity-matrix/"
-
-
- // this creates one part-0000 file in the directory
- mahoutCtx.parallelize(TextDocs).coalesce(1, shuffle=true).saveAsTextFile(inDir)
-
- // to change from using part files to a single .tsv file we'll need to use HDFS
- val fs = FileSystem.get(new Configuration())
- //rename part-00000 to something.tsv
- fs.rename(new Path(inDir + "part-00000"), new Path(inPath))
-
- // local multi-threaded Spark with default HDFS
- RowSimilarityDriver.main(Array(
- "--input", inPath,
- "--output", outPath,
- "--maxSimilaritiesPerRow", "4", // would get all docs similar if we didn't limit them
- "--master", masterUrl))
-
- val simLines = mahoutCtx.textFile(outPath).collect
- tokenize(simLines) should contain theSameElementsAs simDocsTokens
- }
-
- // convert into an Iterable of tokens for 'should contain theSameElementsAs Iterable'
- def tokenize(a: Iterable[String], splitString: String = "[\t ]"): Iterable[String] = {
- var r: Iterable[String] = Iterable()
- a.foreach ( l => r = r ++ l.split(splitString) )
- r
- }
-
- override protected def beforeAll(configMap: ConfigMap) {
- super.beforeAll(configMap)
- RowSimilarityDriver.useContext(mahoutCtx)
- }
-
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.mahout.drivers
+//
+//import org.apache.hadoop.conf.Configuration
+//import org.apache.hadoop.fs.{FileSystem, Path}
+//import org.apache.mahout.math.drm.RLikeDrmOps._
+//import org.apache.mahout.math.drm._
+//import org.apache.mahout.math.scalabindings.RLikeOps._
+//import org.apache.mahout.math.scalabindings._
+//import org.apache.mahout.sparkbindings._
+//import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
+//import org.scalatest.{ConfigMap, FunSuite}
+//
+//
+//class RowSimilarityDriverSuite extends FunSuite with DistributedSparkSuite {
+//
+// val TextDocs = Array(
+// "doc1\tNow is the time for all good people to come to aid of their party",
+// "doc2\tNow is the time for all good people to come to aid of their country",
+// "doc3\tNow is the time for all good people to come to aid of their hood",
+// "doc4\tNow is the time for all good people to come to aid of their friends",
+// "doc5\tNow is the time for all good people to come to aid of their looser brother",
+// "doc6\tThe quick brown fox jumped over the lazy dog",
+// "doc7\tThe quick brown fox jumped over the lazy boy",
+// "doc8\tThe quick brown fox jumped over the lazy cat",
+// "doc9\tThe quick brown fox jumped over the lazy wolverine",
+// "doc10\tThe quick brown fox jumped over the lazy cantelope")// yes that's spelled correctly.
+//
+// test("RowSimilarityDriver text docs no strengths") {
+//
+// val firstFiveSimDocsTokens = tokenize(Iterable(
+// "doc1\tdoc3 doc2 doc4 doc5"))
+//
+// val lastFiveSimDocsTokens = tokenize(Iterable(
+// "doc6\tdoc8 doc10 doc7 doc9"))
+//
+// val inDir = TmpDir + "in-dir/"
+// val inFilename = "in-file.tsv"
+// val inPath = inDir + inFilename
+//
+// val outPath = TmpDir + "similarity-matrices/"
+//
+//
+// // this creates one part-0000 file in the directory
+// mahoutCtx.parallelize(TextDocs).coalesce(1, shuffle=true).saveAsTextFile(inDir)
+//
+// // to change from using part files to a single .tsv file we'll need to use HDFS
+// val fs = FileSystem.get(new Configuration())
+// //rename part-00000 to something.tsv
+// fs.rename(new Path(inDir + "part-00000"), new Path(inPath))
+//
+// // local multi-threaded Spark with default HDFS
+// RowSimilarityDriver.main(Array(
+// "--input", inPath,
+// "--output", outPath,
+// "--omitStrength",
+// "--maxSimilaritiesPerRow", "4", // would get all docs similar if we didn't limit them
+// "--master", masterUrl))
+//
+// val simLines = mahoutCtx.textFile(outPath).collect
+// simLines.foreach { line =>
+// val lineTokens = line.split("[\t ]")
+// if (lineTokens.contains("doc1") ) // docs are two flavors so if only 4 similarities it will effectively classify
+// lineTokens should contain theSameElementsAs firstFiveSimDocsTokens
+// else
+// lineTokens should contain theSameElementsAs lastFiveSimDocsTokens
+// }
+//
+// }
+//
+// test("RowSimilarityDriver text docs") {
+//
+// val simDocsTokens = tokenize(Iterable(
+// "doc1\tdoc3:27.87301122947484 doc2:27.87301122947484 doc4:27.87301122947484 doc5:23.42278065550721",
+// "doc2\tdoc4:27.87301122947484 doc3:27.87301122947484 doc1:27.87301122947484 doc5:23.42278065550721",
+// "doc3\tdoc4:27.87301122947484 doc2:27.87301122947484 doc1:27.87301122947484 doc5:23.42278065550721",
+// "doc4\tdoc3:27.87301122947484 doc2:27.87301122947484 doc1:27.87301122947484 doc5:23.42278065550721",
+// "doc5\tdoc4:23.42278065550721 doc2:23.42278065550721 doc3:23.42278065550721 doc1:23.42278065550721",
+// "doc6\tdoc8:22.936393049704463 doc10:22.936393049704463 doc7:22.936393049704463 doc9:22.936393049704463",
+// "doc7\tdoc6:22.936393049704463 doc8:22.936393049704463 doc10:22.936393049704463 doc9:22.936393049704463",
+// "doc8\tdoc6:22.936393049704463 doc10:22.936393049704463 doc7:22.936393049704463 doc9:22.936393049704463",
+// "doc9\tdoc6:22.936393049704463 doc8:22.936393049704463 doc10:22.936393049704463 doc7:22.936393049704463",
+// "doc10\tdoc6:22.936393049704463 doc8:22.936393049704463 doc7:22.936393049704463 doc9:22.936393049704463"))
+//
+// val inDir = TmpDir + "in-dir/"
+// val inFilename = "in-file.tsv"
+// val inPath = inDir + inFilename
+//
+// val outPath = TmpDir + "similarity-matrix/"
+//
+//
+// // this creates one part-0000 file in the directory
+// mahoutCtx.parallelize(TextDocs).coalesce(1, shuffle=true).saveAsTextFile(inDir)
+//
+// // to change from using part files to a single .tsv file we'll need to use HDFS
+// val fs = FileSystem.get(new Configuration())
+// //rename part-00000 to something.tsv
+// fs.rename(new Path(inDir + "part-00000"), new Path(inPath))
+//
+// // local multi-threaded Spark with default HDFS
+// RowSimilarityDriver.main(Array(
+// "--input", inPath,
+// "--output", outPath,
+// "--maxSimilaritiesPerRow", "4", // would get all docs similar if we didn't limit them
+// "--master", masterUrl))
+//
+// val simLines = mahoutCtx.textFile(outPath).collect
+// tokenize(simLines) should contain theSameElementsAs simDocsTokens
+// }
+//
+// // convert into an Iterable of tokens for 'should contain theSameElementsAs Iterable'
+// def tokenize(a: Iterable[String], splitString: String = "[\t ]"): Iterable[String] = {
+// var r: Iterable[String] = Iterable()
+// a.foreach ( l => r = r ++ l.split(splitString) )
+// r
+// }
+//
+// override protected def beforeAll(configMap: ConfigMap) {
+// super.beforeAll(configMap)
+// RowSimilarityDriver.useContext(mahoutCtx)
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/test/scala/org/apache/mahout/drivers/TextDelimitedReaderWriterSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/drivers/TextDelimitedReaderWriterSuite.scala b/spark/src/test/scala/org/apache/mahout/drivers/TextDelimitedReaderWriterSuite.scala
index 5d92cca..8e56f1e 100644
--- a/spark/src/test/scala/org/apache/mahout/drivers/TextDelimitedReaderWriterSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/drivers/TextDelimitedReaderWriterSuite.scala
@@ -1,53 +1,53 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.mahout.drivers
-
-import org.apache.mahout.math.indexeddataset.DefaultIndexedDatasetReadSchema
-import org.apache.mahout.sparkbindings._
-import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
-import org.scalatest.FunSuite
-
-import scala.collection.JavaConversions._
-
-class TextDelimitedReaderWriterSuite extends FunSuite with DistributedSparkSuite {
- test("indexedDatasetDFSRead should read sparse matrix file with null rows") {
- val OutFile = TmpDir + "similarity-matrices/part-00000"
-
- val lines = Array(
- "galaxy\tnexus:1.0",
- "ipad\tiphone:2.0",
- "nexus\tgalaxy:3.0",
- "iphone\tipad:4.0",
- "surface"
- )
- val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(OutFile)
-
- val data = mahoutCtx.indexedDatasetDFSRead(OutFile, DefaultIndexedDatasetReadSchema)
-
- data.rowIDs.toMap.keySet should equal(Set("galaxy", "ipad", "nexus", "iphone", "surface"))
- data.columnIDs.toMap.keySet should equal(Set("nexus", "iphone", "galaxy", "ipad"))
-
- val a = data.matrix.collect
- a.setRowLabelBindings(mapAsJavaMap(data.rowIDs.toMap).asInstanceOf[java.util.Map[java.lang.String, java.lang.Integer]])
- a.setColumnLabelBindings(mapAsJavaMap(data.columnIDs.toMap).asInstanceOf[java.util.Map[java.lang.String, java.lang.Integer]])
- a.get("galaxy", "nexus") should equal(1.0)
- a.get("ipad", "iphone") should equal(2.0)
- a.get("nexus", "galaxy") should equal(3.0)
- a.get("iphone", "ipad") should equal(4.0)
- }
-}
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with
+// * this work for additional information regarding copyright ownership.
+// * The ASF licenses this file to You under the Apache License, Version 2.0
+// * (the "License"); you may not use this file except in compliance with
+// * the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS,
+// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// * See the License for the specific language governing permissions and
+// * limitations under the License.
+// */
+//
+//package org.apache.mahout.drivers
+//
+//import org.apache.mahout.math.indexeddataset.DefaultIndexedDatasetReadSchema
+//import org.apache.mahout.sparkbindings._
+//import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
+//import org.scalatest.FunSuite
+//
+//import scala.collection.JavaConversions._
+//
+//class TextDelimitedReaderWriterSuite extends FunSuite with DistributedSparkSuite {
+// test("indexedDatasetDFSRead should read sparse matrix file with null rows") {
+// val OutFile = TmpDir + "similarity-matrices/part-00000"
+//
+// val lines = Array(
+// "galaxy\tnexus:1.0",
+// "ipad\tiphone:2.0",
+// "nexus\tgalaxy:3.0",
+// "iphone\tipad:4.0",
+// "surface"
+// )
+// val linesRdd = mahoutCtx.parallelize(lines).saveAsTextFile(OutFile)
+//
+// val data = mahoutCtx.indexedDatasetDFSRead(OutFile, DefaultIndexedDatasetReadSchema)
+//
+// data.rowIDs.toMap.keySet should equal(Set("galaxy", "ipad", "nexus", "iphone", "surface"))
+// data.columnIDs.toMap.keySet should equal(Set("nexus", "iphone", "galaxy", "ipad"))
+//
+// val a = data.matrix.collect
+// a.setRowLabelBindings(mapAsJavaMap(data.rowIDs.toMap).asInstanceOf[java.util.Map[java.lang.String, java.lang.Integer]])
+// a.setColumnLabelBindings(mapAsJavaMap(data.columnIDs.toMap).asInstanceOf[java.util.Map[java.lang.String, java.lang.Integer]])
+// a.get("galaxy", "nexus") should equal(1.0)
+// a.get("ipad", "iphone") should equal(2.0)
+// a.get("nexus", "galaxy") should equal(3.0)
+// a.get("iphone", "ipad") should equal(4.0)
+// }
+//}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/test/scala/org/apache/mahout/sparkbindings/SparkBindingsSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/sparkbindings/SparkBindingsSuite.scala b/spark/src/test/scala/org/apache/mahout/sparkbindings/SparkBindingsSuite.scala
index 61244a1..dece685 100644
--- a/spark/src/test/scala/org/apache/mahout/sparkbindings/SparkBindingsSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/sparkbindings/SparkBindingsSuite.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.mahout.sparkbindings
import java.io.{Closeable, File}
@@ -8,9 +25,7 @@ import org.scalatest.FunSuite
import scala.collection._
-/**
- * @author dmitriy
- */
+
class SparkBindingsSuite extends FunSuite with DistributedSparkSuite {
// This test will succeed only when MAHOUT_HOME is set in the environment. So we keep it for
@@ -26,7 +41,8 @@ class SparkBindingsSuite extends FunSuite with DistributedSparkSuite {
}
mahoutJars.size should be > 0
- mahoutJars.size shouldBe 4
+ // this will depend on the viennacl profile.
+ // mahoutJars.size shouldBe 4
} finally {
IOUtilsScala.close(closeables)
}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/spark/src/test/scala/org/apache/mahout/sparkbindings/test/DistributedSparkSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/sparkbindings/test/DistributedSparkSuite.scala b/spark/src/test/scala/org/apache/mahout/sparkbindings/test/DistributedSparkSuite.scala
index 4c75e75..48d84f8 100644
--- a/spark/src/test/scala/org/apache/mahout/sparkbindings/test/DistributedSparkSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/sparkbindings/test/DistributedSparkSuite.scala
@@ -33,7 +33,7 @@ trait DistributedSparkSuite extends DistributedMahoutSuite with LoggerConfigurat
protected var masterUrl = null.asInstanceOf[String]
protected def initContext() {
- masterUrl = System.getProperties.getOrElse("test.spark.master", "local[3]")
+ masterUrl = System.getProperties.getOrElse("test.spark.master", "local[1]")
val isLocal = masterUrl.startsWith("local")
mahoutCtx = mahoutSparkContext(masterUrl = this.masterUrl,
appName = "MahoutUnitTests",
[2/5] mahout git commit: MAHOUT-1885: Inital commit of VCL bindings.
closes apache/mahout#269 closes apache/mahout#261
Posted by ap...@apache.org.
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/package.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/package.scala b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/package.scala
new file mode 100644
index 0000000..89af010
--- /dev/null
+++ b/viennacl-omp/src/main/scala/org/apache/mahout/viennacl/openmp/package.scala
@@ -0,0 +1,434 @@
+package org.apache.mahout.viennacl
+
+import java.nio._
+
+import org.apache.mahout.math._
+import scalabindings._
+import RLikeOps._
+
+import scala.collection.JavaConversions._
+import org.apache.mahout.viennacl.openmp.javacpp.DenseRowMatrix
+import org.apache.mahout.viennacl.openmp.javacpp._
+import org.bytedeco.javacpp.{DoublePointer, IntPointer}
+
+
+
+package object openmp {
+
+ type IntConvertor = Int => Int
+
+ def toVclDenseRM(src: Matrix, vclCtx: Context = new Context(Context.MAIN_MEMORY)): DenseRowMatrix = {
+ vclCtx.memoryType match {
+ case Context.MAIN_MEMORY \u21d2
+ val vclMx = new DenseRowMatrix(
+ data = repackRowMajor(src, src.nrow, src.ncol),
+ nrow = src.nrow,
+ ncol = src.ncol,
+ ctx = vclCtx
+ )
+ vclMx
+ case _ \u21d2
+ val vclMx = new DenseRowMatrix(src.nrow, src.ncol, vclCtx)
+ fastCopy(src, vclMx)
+ vclMx
+ }
+ }
+
+
+ /**
+ * Convert a dense row VCL matrix to mahout matrix.
+ *
+ * @param src
+ * @return
+ */
+ def fromVclDenseRM(src: DenseRowMatrix): Matrix = {
+ val nrowIntern = src.internalnrow
+ val ncolIntern = src.internalncol
+
+ // A technical debt here:
+
+ // We do double copying here, this is obviously suboptimal, but hopefully we'll compensate
+ // this with gains from running superlinear algorithms in VCL.
+ val dbuff = new DoublePointer(nrowIntern * ncolIntern)
+ Functions.fastCopy(src, dbuff)
+ var srcOffset = 0
+ val ncol = src.ncol
+ val rows = for (irow \u2190 0 until src.nrow) yield {
+
+ val rowvec = new Array[Double](ncol)
+ dbuff.position(srcOffset).get(rowvec)
+
+ srcOffset += ncolIntern
+ rowvec
+ }
+
+ // Always! use shallow = true to avoid yet another copying.
+ new DenseMatrix(rows.toArray, true)
+ }
+
+ def fastCopy(mxSrc: Matrix, dst: DenseRowMatrix) = {
+ val nrowIntern = dst.internalnrow
+ val ncolIntern = dst.internalncol
+
+ assert(nrowIntern >= mxSrc.nrow && ncolIntern >= mxSrc.ncol)
+
+ val rmajorData = repackRowMajor(mxSrc, nrowIntern, ncolIntern)
+ Functions.fastCopy(rmajorData, new DoublePointer(rmajorData).position(rmajorData.limit()), dst)
+
+ rmajorData.close()
+ }
+
+ private def repackRowMajor(mx: Matrix, nrowIntern: Int, ncolIntern: Int): DoublePointer = {
+
+ assert(mx.nrow <= nrowIntern && mx.ncol <= ncolIntern)
+
+ val dbuff = new DoublePointer(nrowIntern * ncolIntern)
+
+ mx match {
+ case dm: DenseMatrix \u21d2
+ val valuesF = classOf[DenseMatrix].getDeclaredField("values")
+ valuesF.setAccessible(true)
+ val values = valuesF.get(dm).asInstanceOf[Array[Array[Double]]]
+ var dstOffset = 0
+ for (irow \u2190 0 until mx.nrow) {
+ val rowarr = values(irow)
+ dbuff.position(dstOffset).put(rowarr, 0, rowarr.size min ncolIntern)
+ dstOffset += ncolIntern
+ }
+ dbuff.position(0)
+ case _ \u21d2
+ // Naive copying. Could be sped up for a DenseMatrix. TODO.
+ for (row \u2190 mx) {
+ val dstOffset = row.index * ncolIntern
+ for (el \u2190 row.nonZeroes) dbuff.put(dstOffset + el.index, el)
+ }
+ }
+
+ dbuff
+ }
+
+ /**
+ *
+ * @param mxSrc
+ * @param ctx
+ * @return
+ */
+ def toVclCmpMatrixAlt(mxSrc: Matrix, ctx: Context): CompressedMatrix = {
+
+ // use repackCSR(matrix, ctx) to convert all ints to unsigned ints if Context is Ocl
+ // val (jumpers, colIdcs, els) = repackCSRAlt(mxSrc)
+ val (jumpers, colIdcs, els) = repackCSR(mxSrc, ctx)
+
+ val compMx = new CompressedMatrix(mxSrc.nrow, mxSrc.ncol, els.capacity().toInt, ctx)
+ compMx.set(jumpers, colIdcs, els, mxSrc.nrow, mxSrc.ncol, els.capacity().toInt)
+ compMx
+ }
+
+ private def repackCSRAlt(mx: Matrix): (IntPointer, IntPointer, DoublePointer) = {
+ val nzCnt = mx.map(_.getNumNonZeroElements).sum
+ val jumpers = new IntPointer(mx.nrow + 1L)
+ val colIdcs = new IntPointer(nzCnt + 0L)
+ val els = new DoublePointer(nzCnt)
+ var posIdx = 0
+
+ var sortCols = false
+
+ // Row-wise loop. Rows may not necessarily come in order. But we have to have them in-order.
+ for (irow \u2190 0 until mx.nrow) {
+
+ val row = mx(irow, ::)
+ jumpers.put(irow.toLong, posIdx)
+
+ // Remember row start index in case we need to restart conversion of this row if out-of-order
+ // column index is detected
+ val posIdxStart = posIdx
+
+ // Retry loop: normally we are done in one pass thru it unless we need to re-run it because
+ // out-of-order column was detected.
+ var done = false
+ while (!done) {
+
+ // Is the sorting mode on?
+ if (sortCols) {
+
+ // Sorting of column indices is on. So do it.
+ row.nonZeroes()
+ // Need to convert to a strict collection out of iterator
+ .map(el \u21d2 el.index \u2192 el.get)
+ // Sorting requires Sequence api
+ .toSeq
+ // Sort by column index
+ .sortBy(_._1)
+ // Flush to the CSR buffers.
+ .foreach { case (index, v) \u21d2
+ colIdcs.put(posIdx.toLong, index)
+ els.put(posIdx.toLong, v)
+ posIdx += 1
+ }
+
+ // Never need to retry if we are already in the sorting mode.
+ done = true
+
+ } else {
+
+ // Try to run unsorted conversion here, switch lazily to sorted if out-of-order column is
+ // detected.
+ var lastCol = 0
+ val nzIter = row.nonZeroes().iterator()
+ var abortNonSorted = false
+
+ while (nzIter.hasNext && !abortNonSorted) {
+
+ val el = nzIter.next()
+ val index = el.index
+
+ if (index < lastCol) {
+
+ // Out of order detected: abort inner loop, reset posIdx and retry with sorting on.
+ abortNonSorted = true
+ sortCols = true
+ posIdx = posIdxStart
+
+ } else {
+
+ // Still in-order: save element and column, continue.
+ els.put(posIdx, el)
+ colIdcs.put(posIdx.toLong, index)
+ posIdx += 1
+
+ // Remember last column seen.
+ lastCol = index
+ }
+ } // inner non-sorted
+
+ // Do we need to re-run this row with sorting?
+ done = !abortNonSorted
+
+ } // if (sortCols)
+
+ } // while (!done) retry loop
+
+ } // row-wise loop
+
+ // Make sure Mahout matrix did not cheat on non-zero estimate.
+ assert(posIdx == nzCnt)
+
+ jumpers.put(mx.nrow.toLong, nzCnt)
+
+ (jumpers, colIdcs, els)
+ }
+
+ // same as repackCSRAlt except converts to jumpers, colIdcs to unsigned ints before setting
+ private def repackCSR(mx: Matrix, context: Context): (IntPointer, IntPointer, DoublePointer) = {
+ val nzCnt = mx.map(_.getNumNonZeroElements).sum
+ val jumpers = new IntPointer(mx.nrow + 1L)
+ val colIdcs = new IntPointer(nzCnt + 0L)
+ val els = new DoublePointer(nzCnt)
+ var posIdx = 0
+
+ var sortCols = false
+
+ def convertInt: IntConvertor = if(context.memoryType == Context.OPENCL_MEMORY) {
+ int2cl_uint
+ } else {
+ i: Int => i: Int
+ }
+
+ // Row-wise loop. Rows may not necessarily come in order. But we have to have them in-order.
+ for (irow \u2190 0 until mx.nrow) {
+
+ val row = mx(irow, ::)
+ jumpers.put(irow.toLong, posIdx)
+
+ // Remember row start index in case we need to restart conversion of this row if out-of-order
+ // column index is detected
+ val posIdxStart = posIdx
+
+ // Retry loop: normally we are done in one pass thru it unless we need to re-run it because
+ // out-of-order column was detected.
+ var done = false
+ while (!done) {
+
+ // Is the sorting mode on?
+ if (sortCols) {
+
+ // Sorting of column indices is on. So do it.
+ row.nonZeroes()
+ // Need to convert to a strict collection out of iterator
+ .map(el \u21d2 el.index \u2192 el.get)
+ // Sorting requires Sequence api
+ .toIndexedSeq
+ // Sort by column index
+ .sortBy(_._1)
+ // Flush to the CSR buffers.
+ .foreach { case (index, v) \u21d2
+ // convert to cl_uint if context is OCL
+ colIdcs.put(posIdx.toLong, convertInt(index))
+ els.put(posIdx.toLong, v)
+ posIdx += 1
+ }
+
+ // Never need to retry if we are already in the sorting mode.
+ done = true
+
+ } else {
+
+ // Try to run unsorted conversion here, switch lazily to sorted if out-of-order column is
+ // detected.
+ var lastCol = 0
+ val nzIter = row.nonZeroes().iterator()
+ var abortNonSorted = false
+
+ while (nzIter.hasNext && !abortNonSorted) {
+
+ val el = nzIter.next()
+ val index = el.index
+
+ if (index < lastCol) {
+
+ // Out of order detected: abort inner loop, reset posIdx and retry with sorting on.
+ abortNonSorted = true
+ sortCols = true
+ posIdx = posIdxStart
+
+ } else {
+
+ // Still in-order: save element and column, continue.
+ els.put(posIdx, el)
+ // convert to cl_uint if context is OCL
+ colIdcs.put(posIdx.toLong, convertInt(index))
+ posIdx += 1
+
+ // Remember last column seen.
+ lastCol = index
+ }
+ } // inner non-sorted
+
+ // Do we need to re-run this row with sorting?
+ done = !abortNonSorted
+
+ } // if (sortCols)
+
+ } // while (!done) retry loop
+
+ } // row-wise loop
+
+ // Make sure Mahout matrix did not cheat on non-zero estimate.
+ assert(posIdx == nzCnt)
+
+ // convert to cl_uint if context is OCL
+ jumpers.put(mx.nrow.toLong, convertInt(nzCnt))
+
+ (jumpers, colIdcs, els)
+ }
+
+
+
+ def fromVclCompressedMatrix(src: CompressedMatrix): Matrix = {
+ val m = src.size1
+ val n = src.size2
+ val NNz = src.nnz
+
+ val row_ptr_handle = src.handle1
+ val col_idx_handle = src.handle2
+ val element_handle = src.handle
+
+ val row_ptr = new IntPointer((m + 1).toLong)
+ val col_idx = new IntPointer(NNz.toLong)
+ val values = new DoublePointer(NNz.toLong)
+
+ Functions.memoryReadInt(row_ptr_handle, 0, (m + 1) * 4, row_ptr, false)
+ Functions.memoryReadInt(col_idx_handle, 0, NNz * 4, col_idx, false)
+ Functions.memoryReadDouble(element_handle, 0, NNz * 8, values, false)
+
+ val rowPtr = row_ptr.asBuffer()
+ val colIdx = col_idx.asBuffer()
+ val vals = values.asBuffer()
+
+ rowPtr.rewind()
+ colIdx.rewind()
+ vals.rewind()
+
+
+ val srMx = new SparseRowMatrix(m, n)
+
+ // read the values back into the matrix
+ var j = 0
+ // row wise, copy any non-zero elements from row(i-1,::)
+ for (i <- 1 to m) {
+ // for each nonzero element, set column col(idx(j) value to vals(j)
+ while (j < rowPtr.get(i)) {
+ srMx(i - 1, colIdx.get(j)) = vals.get(j)
+ j += 1
+ }
+ }
+ srMx
+ }
+
+ def toVclVec(vec: Vector, ctx: Context): VCLVector = {
+
+ vec match {
+ case vec: DenseVector => {
+ val valuesF = classOf[DenseVector].getDeclaredField("values")
+ valuesF.setAccessible(true)
+ val values = valuesF.get(vec).asInstanceOf[Array[Double]]
+ val el_ptr = new DoublePointer(values.length.toLong)
+ el_ptr.put(values, 0, values.length)
+
+ new VCLVector(el_ptr, ctx.memoryType, values.length)
+ }
+
+ case vec: SequentialAccessSparseVector => {
+ val it = vec.iterateNonZero
+ val size = vec.size()
+ val el_ptr = new DoublePointer(size.toLong)
+ while (it.hasNext) {
+ val el: Vector.Element = it.next
+ el_ptr.put(el.index, el.get())
+ }
+ new VCLVector(el_ptr, ctx.memoryType, size)
+ }
+
+ case vec: RandomAccessSparseVector => {
+ val it = vec.iterateNonZero
+ val size = vec.size()
+ val el_ptr = new DoublePointer(size.toLong)
+ while (it.hasNext) {
+ val el: Vector.Element = it.next
+ el_ptr.put(el.index, el.get())
+ }
+ new VCLVector(el_ptr, ctx.memoryType, size)
+ }
+ case _ => throw new IllegalArgumentException("Vector sub-type not supported.")
+ }
+
+ }
+
+ def fromVClVec(vclVec: VCLVector): Vector = {
+ val size = vclVec.size
+ val element_handle = vclVec.handle
+ val ele_ptr = new DoublePointer(size)
+ Functions.memoryReadDouble(element_handle, 0, size * 8, ele_ptr, false)
+
+ // for now just assume its dense since we only have one flavor of
+ // VCLVector
+ val mVec = new DenseVector(size)
+ for (i <- 0 until size) {
+ mVec.setQuick(i, ele_ptr.get(i + 0L))
+ }
+
+ mVec
+ }
+
+
+ // TODO: Fix this? cl_uint must be an unsigned int per each machine's representation of such.
+ // this is currently not working anyways.
+ // cl_uint is needed for OpenCl sparse Buffers
+ // per https://www.khronos.org/registry/cl/sdk/1.1/docs/man/xhtml/scalarDataTypes.html
+ // it is simply an unsigned int, so strip the sign.
+ def int2cl_uint(i: Int): Int = {
+ ((i >>> 1) << 1) + (i & 1)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl-omp/src/test/scala/org/apache/mahout/viennacl/omp/ViennaCLSuiteOMP.scala
----------------------------------------------------------------------
diff --git a/viennacl-omp/src/test/scala/org/apache/mahout/viennacl/omp/ViennaCLSuiteOMP.scala b/viennacl-omp/src/test/scala/org/apache/mahout/viennacl/omp/ViennaCLSuiteOMP.scala
new file mode 100644
index 0000000..8eb3ff9
--- /dev/null
+++ b/viennacl-omp/src/test/scala/org/apache/mahout/viennacl/omp/ViennaCLSuiteOMP.scala
@@ -0,0 +1,249 @@
+package org.apache.mahout.viennacl.openmp
+
+import org.apache.mahout.math._
+import scalabindings._
+import RLikeOps._
+import org.bytedeco.javacpp.DoublePointer
+import org.scalatest.{FunSuite, Matchers}
+import org.apache.mahout.viennacl.openmp.javacpp._
+import org.apache.mahout.viennacl.openmp.javacpp.Functions._
+import org.apache.mahout.viennacl.openmp.javacpp.LinalgFunctions._
+
+import scala.util.Random
+
+class ViennaCLSuiteOMP extends FunSuite with Matchers {
+
+ test("row-major viennacl::matrix") {
+
+ // Just to make sure the javacpp library is loaded:
+ Context.loadLib()
+
+ val m = 20
+ val n = 30
+ val data = new DoublePointer(m * n)
+ val buff = data.asBuffer()
+ // Fill with some noise
+ while (buff.remaining() > 0) buff.put(Random.nextDouble())
+
+ // Create row-major matrix with OpenCL
+ val hostClCtx = new Context(Context.MAIN_MEMORY)
+ val cpuMx = new DenseRowMatrix(data = data, nrow = m, ncol = n, hostClCtx)
+ // And free.
+ cpuMx.close()
+
+ }
+
+
+ test("mmul microbenchmark") {
+ val memCtx = new Context(Context.MAIN_MEMORY)
+
+ val m = 3000
+ val n = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, s)
+ val mxB = new DenseMatrix(s, n)
+
+ // add some data
+ mxA := { (_, _, _) => r.nextDouble() }
+ mxB := { (_, _, _) => r.nextDouble() }
+
+ var ms = System.currentTimeMillis()
+ mxA %*% mxB
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout multiplication time: $ms ms.")
+
+ import LinalgFunctions._
+
+ // openMP/cpu time, including copying:
+ {
+ ms = System.currentTimeMillis()
+ val ompA = toVclDenseRM(mxA, memCtx)
+ val ompB = toVclDenseRM(mxB, memCtx)
+ val ompC = new DenseRowMatrix(prod(ompA, ompB))
+ val mxC = fromVclDenseRM(ompC)
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP multiplication time: $ms ms.")
+
+ ompA.close()
+ ompB.close()
+ ompC.close()
+ }
+
+ }
+
+ test("trans") {
+
+ val ompCtx = new Context(Context.MAIN_MEMORY)
+
+
+ val m = 20
+ val n = 30
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, n)
+
+ // add some data
+ mxA := { (_, _, _) => r.nextDouble() }
+
+
+ // Test transposition in OpenMP
+ {
+ val ompA = toVclDenseRM(src = mxA, ompCtx)
+ val ompAt = new DenseRowMatrix(trans(ompA))
+
+ val mxAt = fromVclDenseRM(ompAt)
+ ompA.close()
+ ompAt.close()
+
+ (mxAt - mxA.t).norm / m / n should be < 1e-16
+ }
+
+ }
+
+ test("sparse mmul microbenchmark") {
+
+ val ompCtx = new Context(Context.MAIN_MEMORY)
+
+ val m = 3000
+ val n = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // sparse row-wise
+ val mxA = new SparseRowMatrix(m, s, false)
+ val mxB = new SparseRowMatrix(s, n, true)
+
+ // add some sparse data with 20% density
+ mxA := { (_, _, v) => if (r.nextDouble() < .20) r.nextDouble() else v }
+ mxB := { (_, _, v) => if (r.nextDouble() < .20) r.nextDouble() else v }
+
+ var ms = System.currentTimeMillis()
+ val mxC = mxA %*% mxB
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout Sparse multiplication time: $ms ms.")
+
+
+ // Test multiplication in OpenMP
+ {
+ ms = System.currentTimeMillis()
+ // val ompA = toVclCompressedMatrix(src = mxA, ompCtx)
+ // val ompB = toVclCompressedMatrix(src = mxB, ompCtx)
+
+ val ompA = toVclCmpMatrixAlt(mxA, ompCtx)
+ val ompB = toVclCmpMatrixAlt(mxB, ompCtx)
+
+ val ompC = new CompressedMatrix(prod(ompA, ompB))
+
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP Sparse multiplication time: $ms ms.")
+
+ val ompMxC = fromVclCompressedMatrix(ompC)
+ (mxC - ompMxC).norm / mxC.nrow / mxC.ncol should be < 1e-16
+
+ ompA.close()
+ ompB.close()
+ ompC.close()
+
+ }
+
+ }
+
+ test("VCL Dense Matrix %*% Dense vector - no OpenCl") {
+
+ val ompCtx = new Context(Context.MAIN_MEMORY)
+
+
+ val m = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxA = new DenseMatrix(m, s)
+ val dvecB = new DenseVector(s)
+
+ // add some random data
+ mxA := { (_,_,_) => r.nextDouble() }
+ dvecB := { (_,_) => r.nextDouble() }
+
+ //test in matrix %*% vec
+ var ms = System.currentTimeMillis()
+ val mDvecC = mxA %*% dvecB
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout dense matrix %*% dense vector multiplication time: $ms ms.")
+
+
+ //Test multiplication in OpenMP
+ {
+
+ ms = System.currentTimeMillis()
+ val ompMxA = toVclDenseRM(mxA, ompCtx)
+ val ompVecB = toVclVec(dvecB, ompCtx)
+
+ val ompVecC = new VCLVector(prod(ompMxA, ompVecB))
+ val ompDvecC = fromVClVec(ompVecC)
+
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP dense matrix %*% dense vector multiplication time: $ms ms.")
+ (ompDvecC.toColMatrix - mDvecC.toColMatrix).norm / s should be < 1e-16
+
+ ompMxA.close()
+ ompVecB.close()
+ ompVecC.close()
+ }
+
+ }
+
+
+ test("Sparse %*% Dense mmul microbenchmark") {
+ val memCtx = new Context(Context.MAIN_MEMORY)
+
+ val m = 3000
+ val n = 3000
+ val s = 1000
+
+ val r = new Random(1234)
+
+ // Dense row-wise
+ val mxSr = new SparseMatrix(m, s)
+ val mxDn = new DenseMatrix(s, n)
+
+ // add some data
+ mxSr := { (_, _, v) => if (r.nextDouble() < .20) r.nextDouble() else v }
+ mxDn := { (_, _, _) => r.nextDouble() }
+
+ var ms = System.currentTimeMillis()
+ mxSr %*% mxDn
+ ms = System.currentTimeMillis() - ms
+ info(s"Mahout multiplication time: $ms ms.")
+
+ import LinalgFunctions._
+
+
+ // openMP/cpu time, including copying:
+ {
+ ms = System.currentTimeMillis()
+ val ompA = toVclCmpMatrixAlt(mxSr, memCtx)
+ val ompB = toVclDenseRM(mxDn, memCtx)
+ val ompC = new DenseRowMatrix(prod(ompA, ompB))
+ val mxC = fromVclDenseRM(ompC)
+ ms = System.currentTimeMillis() - ms
+ info(s"ViennaCL/cpu/OpenMP multiplication time: $ms ms.")
+
+ ompA.close()
+ ompB.close()
+ ompC.close()
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/linux-haswell.properties
----------------------------------------------------------------------
diff --git a/viennacl/linux-haswell.properties b/viennacl/linux-haswell.properties
new file mode 100644
index 0000000..52d5cec
--- /dev/null
+++ b/viennacl/linux-haswell.properties
@@ -0,0 +1,28 @@
+platform=linux-haswell
+platform.path.separator=:
+platform.source.suffix=.cpp
+platform.includepath.prefix=-I
+platform.includepath=
+platform.compiler=g++
+platform.compiler.cpp11=-std=c++11
+platform.compiler.default=
+platform.compiler.fastfpu=-msse3 -ffast-math
+platform.compiler.viennacl=-fopenmp -fpermissive
+platform.compiler.nodeprecated=-Wno-deprecated-declarations
+#build for haswell arch with for GCC >= 4.9.0
+platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=haswell -m64 -Wall -O3 -fPIC -shared -s -o\u0020
+#for GCC < 4.9.0 use -march=core-avx2 for haswell arch
+#platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=core-avx2 -m64 -Wall -Ofast -fPIC -shared -s -o\u0020
+#build for native:
+#platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=native -m64 -Wall -Ofast -fPIC -shared -s -o\u0020
+platform.linkpath.prefix=-L
+platform.linkpath.prefix2=-Wl,-rpath,
+platform.linkpath=
+platform.link.prefix=-l
+platform.link.suffix=
+platform.link=
+platform.framework.prefix=-F
+platform.framework.suffix=
+platform.framework=
+platform.library.prefix=lib
+platform.library.suffix=.so
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/linux-x86_64-viennacl.properties
----------------------------------------------------------------------
diff --git a/viennacl/linux-x86_64-viennacl.properties b/viennacl/linux-x86_64-viennacl.properties
new file mode 100644
index 0000000..e5de1fa
--- /dev/null
+++ b/viennacl/linux-x86_64-viennacl.properties
@@ -0,0 +1,24 @@
+platform=linux-x86_64
+platform.path.separator=:
+platform.source.suffix=.cpp
+platform.includepath.prefix=-I
+platform.includepath=
+platform.compiler=g++
+platform.compiler.cpp11=-std=c++11
+platform.compiler.default=
+platform.compiler.fastfpu=-msse3 -ffast-math
+platform.compiler.viennacl=-fopenmp -fpermissive
+platform.compiler.nodeprecated=-Wno-deprecated-declarations
+# platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=x86-64 -m64 -Wall -O3 -fPIC -shared -s -o\u0020
+platform.compiler.output=-Wl,-rpath,$ORIGIN/ -Wl,-z,noexecstack -Wl,-Bsymbolic -march=native -m64 -Wall -Ofast -fPIC -shared -s -o\u0020
+platform.linkpath.prefix=-L
+platform.linkpath.prefix2=-Wl,-rpath,
+platform.linkpath=
+platform.link.prefix=-l
+platform.link.suffix=
+platform.link=
+platform.framework.prefix=-F
+platform.framework.suffix=
+platform.framework=
+platform.library.prefix=lib
+platform.library.suffix=.so
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/pom.xml
----------------------------------------------------------------------
diff --git a/viennacl/pom.xml b/viennacl/pom.xml
new file mode 100644
index 0000000..bd543f3
--- /dev/null
+++ b/viennacl/pom.xml
@@ -0,0 +1,271 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout</artifactId>
+ <version>0.13.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <artifactId>mahout-native-viennacl_${scala.compat.version}</artifactId>
+
+ <name>Mahout Native VienniaCL OpenCL Bindings</name>
+ <description>Native Structures and interfaces to be used from Mahout math-scala.
+ </description>
+
+ <packaging>jar</packaging>
+
+ <build>
+ <plugins>
+ <!-- create test jar so other modules can reuse the native test utility classes. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ <phase>package</phase>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>add-scala-sources</id>
+ <phase>initialize</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-compile</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>compile</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>scala-test-compile</id>
+ <phase>process-test-resources</phase>
+ <goals>
+ <goal>testCompile</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
+ <!--this is what scalatest recommends to do to enable scala tests -->
+
+ <!-- disable surefire -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>true</skipTests>
+ </configuration>
+ </plugin>
+ <!-- enable scalatest -->
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <argLine>-Xmx4g</argLine>
+ </configuration>
+ </plugin>
+
+
+ <!--JavaCPP native build plugin-->
+ <!-- old-style way to get it to compile. -->
+ <!--based on https://github.com/bytedeco/javacpp/wiki/Maven-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ <executions>
+ <execution>
+ <id>javacpp</id>
+ <phase>process-classes</phase>
+ <goals>
+ <goal>exec</goal>
+ </goals>
+ <configuration>
+ <environmentVariables>
+ <LD_LIBRARY_PATH>{project.basedir}/target/classes/org/apache/mahout/javacpp/linalg/linux-x86_64/
+ </LD_LIBRARY_PATH>
+ </environmentVariables>
+ <executable>java</executable>
+ <arguments>
+ <argument>-jar</argument>
+ <argument>${org.bytedeco:javacpp:jar}</argument>
+ <argument>-propertyfile</argument>
+ <argument>linux-x86_64-viennacl.properties</argument>
+ <argument>-classpath</argument>
+ <argument>${project.build.outputDirectory}:${org.scala-lang:scala-library:jar}</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.CompressedMatrix</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.Context</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.MatrixBase</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.DenseRowMatrix</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.DenseColumnMatrix</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.MatMatProdExpression</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.ProdExpression</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.SrMatDnMatProdExpression</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.MatrixTransExpression</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.LinalgFunctions</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.Functions</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.VectorBase</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.VCLVector</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.VecMultExpression</argument>
+ <argument>org.apache.mahout.viennacl.opencl.javacpp.MemHandle</argument>
+ <argument>org.apache.mahout.viennacl.opencl.GPUMMul</argument>
+ <argument>org.apache.mahout.viennacl.opencl.GPUMMul$</argument>
+ </arguments>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>2.3</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>properties</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>exec-maven-plugin</artifactId>
+ <version>1.2.1</version>
+ </plugin>
+
+ </plugins>
+
+ </build>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>mahout-math-scala_${scala.compat.version}</artifactId>
+ </dependency>
+
+ <!-- 3rd-party -->
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+
+ <!-- scala stuff -->
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.bytedeco</groupId>
+ <artifactId>javacpp</artifactId>
+ <version>1.2.4</version>
+ </dependency>
+
+ </dependencies>
+
+
+ <profiles>
+ <profile>
+ <id>mahout-release</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>generate-scaladoc</id>
+ <goals>
+ <goal>doc</goal>
+ </goals>
+ </execution>
+ <execution>
+ <id>attach-scaladoc-jar</id>
+ <goals>
+ <goal>doc-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>travis</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- Limit memory for unit tests in Travis -->
+ <argLine>-Xmx4g</argLine>
+ <!--<argLine>-Djava.library.path=${project.build.directory}/libs/natives/linux-x86_64:${project.build.directory}/libs/natives/linux:${project.build.directory}/libs/natives/maxosx</argLine>-->
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <configuration>
+ <!-- Limit memory for integration tests in Travis -->
+ <argLine>-Xmx4g</argLine>
+ <!--<argLine>-Djava.library.path=${project.build.directory}/libs/natives/linux-x86_64:${project.build.directory}/libs/natives/linux:${project.build.directory}/libs/natives/maxosx</argLine>-->
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/Functions.java
----------------------------------------------------------------------
diff --git a/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/Functions.java b/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/Functions.java
new file mode 100644
index 0000000..1c14f97
--- /dev/null
+++ b/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/Functions.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.viennacl.opencl.javacpp;
+
+import org.bytedeco.javacpp.BytePointer;
+import org.bytedeco.javacpp.DoublePointer;
+import org.bytedeco.javacpp.IntPointer;
+import org.bytedeco.javacpp.annotation.*;
+
+import java.nio.DoubleBuffer;
+import java.nio.IntBuffer;
+
+
+@Properties(inherit = Context.class,
+ value = @Platform(
+ library = "jniViennaCL"
+ )
+)
+@Namespace("viennacl")
+public final class Functions {
+
+ private Functions() {
+ }
+
+ // This is (imo) an inconsistency in Vienna cl: almost all operations require MatrixBase, and
+ // fast_copy require type `matrix`, i.e., one of DenseRowMatrix or DenseColumnMatrix.
+ @Name("fast_copy")
+ public static native void fastCopy(DoublePointer srcBegin, DoublePointer srcEnd, @ByRef DenseRowMatrix dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(DoublePointer srcBegin, DoublePointer srcEnd, @ByRef DenseColumnMatrix dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@ByRef DenseRowMatrix src, DoublePointer dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@ByRef DenseColumnMatrix src, DoublePointer dst);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@Const @ByRef VectorBase dst, @Const @ByRef VCLVector src);
+
+ @Name("fast_copy")
+ public static native void fastCopy(@Const @ByRef VCLVector src, @Const @ByRef VectorBase dst);
+
+
+ @ByVal
+ public static native MatrixTransExpression trans(@ByRef MatrixBase src);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadInt(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ IntPointer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadDouble(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ DoublePointer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadInt(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ IntBuffer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadDouble(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ DoubleBuffer ptr,
+ boolean async);
+
+ @Name("backend::memory_read")
+ public static native void memoryReadBytes(@Const @ByRef MemHandle src_buffer,
+ int bytes_to_read,
+ int offset,
+ BytePointer ptr,
+ boolean async);
+
+
+ static {
+ Context.loadLib();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/LinalgFunctions.java
----------------------------------------------------------------------
diff --git a/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/LinalgFunctions.java b/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/LinalgFunctions.java
new file mode 100644
index 0000000..9540691
--- /dev/null
+++ b/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/LinalgFunctions.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.viennacl.opencl.javacpp;
+
+import org.bytedeco.javacpp.annotation.*;
+
+
+@Properties(inherit = Context.class,
+ value = @Platform(
+ library = "jniViennaCL"
+ )
+)
+@Namespace("viennacl::linalg")
+public final class LinalgFunctions {
+
+ private LinalgFunctions() {
+ }
+
+ static {
+ Context.loadLib();
+ }
+
+
+ @ByVal
+ public static native MatMatProdExpression prod(@Const @ByRef MatrixBase a,
+ @Const @ByRef MatrixBase b);
+
+ @ByVal
+ public static native ProdExpression prod(@Const @ByRef CompressedMatrix a,
+ @Const @ByRef CompressedMatrix b);
+
+ @ByVal
+ public static native MatVecProdExpression prod(@Const @ByRef MatrixBase a,
+ @Const @ByRef VectorBase b);
+
+ @ByVal
+ public static native SrMatDnMatProdExpression prod(@Const @ByRef CompressedMatrix spMx,
+ @Const @ByRef MatrixBase dMx);
+ @ByVal
+ @Name("prod")
+ public static native DenseColumnMatrix prodCm(@Const @ByRef MatrixBase a,
+ @Const @ByRef MatrixBase b);
+ @ByVal
+ @Name("prod")
+ public static native DenseRowMatrix prodRm(@Const @ByRef MatrixBase a,
+ @Const @ByRef MatrixBase b);
+
+ @ByVal
+ @Name("prod")
+ public static native DenseRowMatrix prodRm(@Const @ByRef CompressedMatrix spMx,
+ @Const @ByRef MatrixBase dMx);
+
+
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseRowMatrix a,
+// @Const @ByRef DenseRowMatrix b);
+//
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseRowMatrix a,
+// @Const @ByRef DenseColumnMatrix b);
+//
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseColumnMatrix a,
+// @Const @ByRef DenseRowMatrix b);
+//
+// @ByVal
+// public static native MatrixProdExpression prod(@Const @ByRef DenseColumnMatrix a,
+// @Const @ByRef DenseColumnMatrix b);
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/MatrixTransExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/MatrixTransExpression.scala b/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/MatrixTransExpression.scala
new file mode 100644
index 0000000..115af05
--- /dev/null
+++ b/viennacl/src/main/java/org/apache/mahout/viennacl/opencl/javacpp/MatrixTransExpression.scala
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp;
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ include = Array("matrix.hpp"),
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::matrix_base<double>, " +
+ "const viennacl::matrix_base<double>, " +
+ "viennacl::op_trans>"))
+class MatrixTransExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/GPUMMul.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/GPUMMul.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/GPUMMul.scala
new file mode 100644
index 0000000..936448d
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/GPUMMul.scala
@@ -0,0 +1,455 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.viennacl.opencl
+
+import org.apache.mahout.logging._
+import org.apache.mahout.math
+import org.apache.mahout.math._
+import org.apache.mahout.math.backend.incore.MMulSolver
+import org.apache.mahout.math.flavor.{BackEnum, TraversingStructureEnum}
+import org.apache.mahout.math.function.Functions
+import org.apache.mahout.math.scalabindings.RLikeOps._
+import org.apache.mahout.math.scalabindings._
+import org.apache.mahout.viennacl.opencl.javacpp.Functions._
+import org.apache.mahout.viennacl.opencl.javacpp.LinalgFunctions._
+import org.apache.mahout.viennacl.opencl.javacpp.{CompressedMatrix, Context, DenseRowMatrix}
+
+import scala.collection.JavaConversions._
+object GPUMMul extends MMBinaryFunc {
+
+ private final implicit val log = getLog(GPUMMul.getClass)
+
+ override def apply(a: Matrix, b: Matrix, r: Option[Matrix]): Matrix = {
+
+ require(a.ncol == b.nrow, "Incompatible matrix sizes in matrix multiplication.")
+
+ val (af, bf) = (a.getFlavor, b.getFlavor)
+ val backs = (af.getBacking, bf.getBacking)
+ val sd = (af.getStructure, math.scalabindings.densityAnalysis(a), bf.getStructure, densityAnalysis(b))
+
+
+ try {
+
+ val alg: MMulAlg = backs match {
+
+ // Both operands are jvm memory backs.
+ case (BackEnum.JVMMEM, BackEnum.JVMMEM) \u21d2
+
+ sd match {
+
+ // Multiplication cases by a diagonal matrix.
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.COLWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagCW
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.SPARSECOLWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagCW
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.ROWWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagRW
+ case (TraversingStructureEnum.VECTORBACKED, _, TraversingStructureEnum.SPARSEROWWISE, _)
+ if a.isInstanceOf[DiagonalMatrix] \u21d2 jvmDiagRW
+
+ case (TraversingStructureEnum.COLWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmCWDiag
+ case (TraversingStructureEnum.SPARSECOLWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmCWDiag
+ case (TraversingStructureEnum.ROWWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmRWDiag
+ case (TraversingStructureEnum.SPARSEROWWISE, _, TraversingStructureEnum.VECTORBACKED, _)
+ if b.isInstanceOf[DiagonalMatrix] \u21d2 jvmRWDiag
+
+ // Dense-dense cases
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.COLWISE, true) if a eq b.t \u21d2 gpuDRWAAt
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.COLWISE, true) if a.t eq b \u21d2 gpuDRWAAt
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.COLWISE, true) \u21d2 gpuRWCW
+ case (TraversingStructureEnum.ROWWISE, true, TraversingStructureEnum.ROWWISE, true) \u21d2 jvmRWRW
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.COLWISE, true) \u21d2 jvmCWCW
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.ROWWISE, true) if a eq b.t \u21d2 jvmDCWAAt
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.ROWWISE, true) if a.t eq b \u21d2 jvmDCWAAt
+ case (TraversingStructureEnum.COLWISE, true, TraversingStructureEnum.ROWWISE, true) \u21d2 jvmCWRW
+
+ // Sparse row matrix x sparse row matrix (array of vectors)
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.ROWWISE, false) \u21d2 gpuSparseRWRW
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.COLWISE, false) \u21d2 jvmSparseRWCW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.ROWWISE, false) \u21d2 jvmSparseCWRW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.COLWISE, false) \u21d2 jvmSparseCWCW
+
+ // Sparse matrix x sparse matrix (hashtable of vectors)
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2
+ gpuSparseRowRWRW
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2
+ jvmSparseRowRWCW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2
+ jvmSparseRowCWRW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2
+ jvmSparseRowCWCW
+
+ // Sparse matrix x non-like
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 gpuSparseRowRWRW
+ case (TraversingStructureEnum.SPARSEROWWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseRowRWCW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 jvmSparseRowCWRW
+ case (TraversingStructureEnum.SPARSECOLWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseCWCW
+ case (TraversingStructureEnum.ROWWISE, _, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2 gpuSparseRWRW
+ case (TraversingStructureEnum.ROWWISE, _, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2 jvmSparseRWCW
+ case (TraversingStructureEnum.COLWISE, _, TraversingStructureEnum.SPARSEROWWISE, false) \u21d2 jvmSparseCWRW
+ case (TraversingStructureEnum.COLWISE, _, TraversingStructureEnum.SPARSECOLWISE, false) \u21d2 jvmSparseRowCWCW
+
+ // Everything else including at least one sparse LHS or RHS argument
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 gpuSparseRWRW
+ case (TraversingStructureEnum.ROWWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseRWCW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.ROWWISE, _) \u21d2 jvmSparseCWRW
+ case (TraversingStructureEnum.COLWISE, false, TraversingStructureEnum.COLWISE, _) \u21d2 jvmSparseCWCW2flips
+
+ // Sparse methods are only effective if the first argument is sparse, so we need to do a swap.
+ case (_, _, _, false) \u21d2 (a, b, r) \u21d2 apply(b.t, a.t, r.map {
+ _.t
+ }).t
+
+ // Default jvm-jvm case.
+ // for some reason a SrarseRowMatrix DRM %*% SrarseRowMatrix DRM was dumping off to here
+ case _ \u21d2 gpuRWCW
+ }
+ }
+
+ alg(a, b, r)
+ } catch {
+ // TODO FASTHACK: just revert to JVM if there is an exception..
+ // eg. java.lang.nullPointerException if more openCL contexts
+ // have been created than number of GPU cards.
+ // better option wuold be to fall back to OpenCl First.
+ case ex: Exception =>
+ println(ex.getMessage + "falling back to JVM MMUL")
+ return MMul(a, b, r)
+ }
+ }
+
+ type MMulAlg = MMBinaryFunc
+
+ @inline
+ private def gpuRWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("gpuRWCW")
+//
+// require(r.forall(mxR \u21d2 mxR.nrow == a.nrow && mxR.ncol == b.ncol))
+// val (m, n) = (a.nrow, b.ncol)
+//
+// val mxR = r.getOrElse(if (densityAnalysis(a)) a.like(m, n) else b.like(m, n))
+//
+// for (row \u2190 0 until mxR.nrow; col \u2190 0 until mxR.ncol) {
+// // this vector-vector should be sort of optimized, right?
+// mxR(row, col) = a(row, ::) dot b(::, col)
+// }
+// mxR
+
+ val hasElementsA = a.zSum() > 0.0
+ val hasElementsB = b.zSum() > 0.0
+
+ // A has a sparse matrix structure of unknown size. We do not want to
+ // simply convert it to a Dense Matrix which may result in an OOM error.
+
+ // If it is empty use JVM MMul, since we can not convert it to a VCL CSR Matrix.
+ if (!hasElementsA) {
+ println("Matrix a has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ // CSR matrices are efficient up to 50% non-zero
+ if(b.getFlavor.isDense) {
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, oclCtx)
+ val oclB = toVclDenseRM(b, oclCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ } else {
+ // Fall back to JVM based MMul if either matrix is sparse and empty
+ if (!hasElementsA || !hasElementsB) {
+ println("Matrix a or b has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, oclCtx)
+ val oclB = toVclCmpMatrixAlt(b, oclCtx)
+ val oclC = new CompressedMatrix(prod(oclA, oclB))
+ val mxC = fromVclCompressedMatrix(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ }
+ }
+
+
+ @inline
+ private def jvmRWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmRWRW")
+ // A bit hackish: currently, this relies a bit on the fact that like produces RW(?)
+ val bclone = b.like(b.ncol, b.nrow).t
+ for (brow \u2190 b) bclone(brow.index(), ::) := brow
+
+ require(bclone.getFlavor.getStructure == TraversingStructureEnum.COLWISE || bclone.getFlavor.getStructure ==
+ TraversingStructureEnum.SPARSECOLWISE, "COL wise conversion assumption of RHS is wrong, do over this code.")
+
+ gpuRWCW(a, bclone, r)
+ }
+
+ private def jvmCWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmCWCW")
+ jvmRWRW(b.t, a.t, r.map(_.t)).t
+ }
+
+ private def jvmCWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmCWRW")
+ // This is a primary contender with Outer Prod sum algo.
+ // Here, we force-reorient both matrices and run RWCW.
+ // A bit hackish: currently, this relies a bit on the fact that clone always produces RW(?)
+ val aclone = a.cloned
+
+ require(aclone.getFlavor.getStructure == TraversingStructureEnum.ROWWISE || aclone.getFlavor.getStructure ==
+ TraversingStructureEnum.SPARSEROWWISE, "Row wise conversion assumption of RHS is wrong, do over this code.")
+
+ jvmRWRW(aclone, b, r)
+ }
+
+ // left is Sparse right is any
+ private def gpuSparseRWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("gpuSparseRWRW")
+ val mxR = r.getOrElse(b.like(a.nrow, b.ncol))
+
+
+// // This is basically almost the algorithm from SparseMatrix.times
+// for (arow \u2190 a; ael \u2190 arow.nonZeroes)
+// mxR(arow.index(), ::).assign(b(ael.index, ::), Functions.plusMult(ael))
+//
+// mxR
+
+ // make sure that the matrix is not empty. VCL {{compressed_matrix}}s must
+ // hav nnz > 0
+ // this method is horribly inefficent. however there is a difference between
+ // getNumNonDefaultElements() and getNumNonZeroElements() which we do not always
+ // have access to created MAHOUT-1882 for this
+ val hasElementsA = a.zSum() > 0.0
+ val hasElementsB = b.zSum() > 0.0
+
+ // A has a sparse matrix structure of unknown size. We do not want to
+ // simply convert it to a Dense Matrix which may result in an OOM error.
+ // If it is empty use JVM MMul, since we can not convert it to a VCL CSR Matrix.
+ if (!hasElementsA) {
+ println("Matrix a has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ // CSR matrices are efficient up to 50% non-zero
+ if(b.getFlavor.isDense) {
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, oclCtx)
+ val oclB = toVclDenseRM(b, oclCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ } else {
+ // Fall back to JVM based MMul if either matrix is sparse and empty
+ if (!hasElementsA || !hasElementsB) {
+ println("Matrix a or b has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, oclCtx)
+ val oclB = toVclCmpMatrixAlt(b, oclCtx)
+ val oclC = new CompressedMatrix(prod(oclA, oclB))
+ val mxC = fromVclCompressedMatrix(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ }
+
+ }
+
+ //sparse %*% dense
+ private def gpuSparseRowRWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("gpuSparseRowRWRW")
+ val hasElementsA = a.zSum() > 0
+
+ // A has a sparse matrix structure of unknown size. We do not want to
+ // simply convert it to a Dense Matrix which may result in an OOM error.
+ // If it is empty fall back to JVM MMul, since we can not convert it
+ // to a VCL CSR Matrix.
+ if (!hasElementsA) {
+ println("Matrix a has zero elements can not convert to CSR")
+ return MMul(a, b, r)
+ }
+
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val oclA = toVclCmpMatrixAlt(a, oclCtx)
+ val oclB = toVclDenseRM(b, oclCtx)
+ val oclC = new DenseRowMatrix(prod(oclA, oclB))
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ oclB.close()
+ oclC.close()
+
+ mxC
+ }
+
+ private def jvmSparseRowCWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ gpuSparseRowRWRW(b.t, a.t, r.map(_.t)).t
+
+ private def jvmSparseRowCWCW2flips(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ gpuSparseRowRWRW(a cloned, b cloned, r)
+
+ private def jvmSparseRowRWCW(a: Matrix, b: Matrix, r: Option[Matrix]) =
+ gpuSparseRowRWRW(a, b cloned, r)
+
+
+ private def jvmSparseRowCWRW(a: Matrix, b: Matrix, r: Option[Matrix]) =
+ gpuSparseRowRWRW(a cloned, b, r)
+
+ private def jvmSparseRWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ gpuSparseRWRW(a, b.cloned, r)
+
+ private def jvmSparseCWRW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ gpuSparseRWRW(a cloned, b, r)
+
+ private def jvmSparseCWCW(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ gpuSparseRWRW(b.t, a.t, r.map(_.t)).t
+
+ private def jvmSparseCWCW2flips(a: Matrix, b: Matrix, r: Option[Matrix] = None) =
+ gpuSparseRWRW(a cloned, b cloned, r)
+
+ private def jvmDiagRW(diagm:Matrix, b:Matrix, r:Option[Matrix] = None):Matrix = {
+ println("jvmDiagRW")
+ val mxR = r.getOrElse(b.like(diagm.nrow, b.ncol))
+
+ for (del \u2190 diagm.diagv.nonZeroes())
+ mxR(del.index, ::).assign(b(del.index, ::), Functions.plusMult(del))
+
+ mxR
+ }
+
+ private def jvmDiagCW(diagm: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmDiagCW")
+ val mxR = r.getOrElse(b.like(diagm.nrow, b.ncol))
+ for (bcol \u2190 b.t) mxR(::, bcol.index()) := bcol * diagm.diagv
+ mxR
+ }
+
+ private def jvmCWDiag(a: Matrix, diagm: Matrix, r: Option[Matrix] = None) =
+ jvmDiagRW(diagm, a.t, r.map {_.t}).t
+
+ private def jvmRWDiag(a: Matrix, diagm: Matrix, r: Option[Matrix] = None) =
+ jvmDiagCW(diagm, a.t, r.map {_.t}).t
+
+
+ /** Dense column-wise AA' */
+ private def jvmDCWAAt(a:Matrix, b:Matrix, r:Option[Matrix] = None) = {
+ // a.t must be equiv. to b. Cloning must rewrite to row-wise.
+ gpuDRWAAt(a.cloned,null,r)
+ }
+
+ /** Dense Row-wise AA' */
+ // we probably will not want to use this for the actual release unless A is cached already
+ // but adding for testing purposes.
+ private def gpuDRWAAt(a:Matrix, b:Matrix, r:Option[Matrix] = None) = {
+ // a.t must be equiv to b.
+ println("executing on gpu")
+ debug("AAt computation detected; passing off to GPU")
+
+ // Check dimensions if result is supplied.
+ require(r.forall(mxR \u21d2 mxR.nrow == a.nrow && mxR.ncol == a.nrow))
+
+ val mxR = r.getOrElse(a.like(a.nrow, a.nrow))
+
+ var ms = System.currentTimeMillis()
+ val oclCtx = new Context(Context.OPENCL_MEMORY)
+ val oclA = toVclDenseRM(src = a, oclCtx)
+ val oclAt = new DenseRowMatrix(trans(oclA))
+ val oclC = new DenseRowMatrix(prod(oclA, oclAt))
+
+ val mxC = fromVclDenseRM(oclC)
+ ms = System.currentTimeMillis() - ms
+ debug(s"ViennaCL/OpenCL multiplication time: $ms ms.")
+
+ oclA.close()
+ //oclApr.close()
+ oclAt.close()
+ oclC.close()
+
+ mxC
+
+ }
+
+ private def jvmOuterProdSum(a: Matrix, b: Matrix, r: Option[Matrix] = None): Matrix = {
+ println("jvmOuterProdSum")
+ // This may be already laid out for outer product computation, which may be faster than reorienting
+ // both matrices? need to check.
+ val (m, n) = (a.nrow, b.ncol)
+
+ // Prefer col-wise result iff a is dense and b is sparse. In all other cases default to row-wise.
+ val preferColWiseR = a.getFlavor.isDense && !b.getFlavor.isDense
+
+ val mxR = r.getOrElse {
+ (a.getFlavor.isDense, preferColWiseR) match {
+ case (false, false) \u21d2 b.like(m, n)
+ case (false, true) \u21d2 b.like(n, m).t
+ case (true, false) \u21d2 a.like(m, n)
+ case (true, true) \u21d2 a.like(n, m).t
+ }
+ }
+
+ // Loop outer products
+ if (preferColWiseR) {
+ // this means B is sparse and A is not, so we need to iterate over b values and update R columns with +=
+ // one at a time.
+ for ((acol, brow) \u2190 a.t.zip(b); bel \u2190 brow.nonZeroes) mxR(::, bel.index()) += bel * acol
+ } else {
+ for ((acol, brow) \u2190 a.t.zip(b); ael \u2190 acol.nonZeroes()) mxR(ael.index(), ::) += ael * brow
+ }
+
+ mxR
+ }
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/CompressedMatrix.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/CompressedMatrix.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/CompressedMatrix.scala
new file mode 100644
index 0000000..5a84ac5
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/CompressedMatrix.scala
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import java.nio._
+
+import org.bytedeco.javacpp._
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ include = Array("compressed_matrix.hpp"),
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::compressed_matrix<double>"))
+final class CompressedMatrix(defaultCtr: Boolean = true) extends Pointer {
+
+ protected val ptrs = new ArrayBuffer[Pointer]()
+
+ // call this after set or better TODO: yet wrap set() in a public method that will call this
+ def registerPointersForDeallocation(p:Pointer): Unit = {
+ ptrs += p
+ }
+
+ override def deallocate(deallocate: Boolean): Unit = {
+ super.deallocate(deallocate)
+ ptrs.foreach(_.close())
+ }
+
+ if (defaultCtr) allocate()
+
+ def this(nrow: Int, ncol: Int, ctx: Context = new Context) {
+ this(false)
+ allocate(nrow, ncol, ctx)
+ }
+
+ def this(nrow: Int, ncol: Int, nonzeros: Int, ctx: Context = new Context) {
+ this(false)
+ allocate(nrow, ncol, nonzeros, ctx)
+ }
+
+ def this(pe: ProdExpression) {
+ this(false)
+ allocate(pe)
+ }
+
+ @native protected def allocate()
+
+ @native protected def allocate(nrow: Int, ncol: Int, nonzeros: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(nrow: Int, ncol: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(@Const @ByRef pe: ProdExpression)
+
+// @native protected def allocate(db: DoubleBuffer)
+//
+// @native protected def allocate(ib: IntBuffer)
+
+ // Warning: apparently there are differences in bit interpretation between OpenCL and everything
+ // else for unsigned int type. So, for OpenCL backend, rowJumper and colIndices have to be packed
+ // with reference to that cl_uint type that Vienna-CL defines.
+ @native def set(@Cast(Array("const void*")) rowJumper: IntBuffer,
+ @Cast(Array("const void*")) colIndices: IntBuffer,
+ @Const elements: DoubleBuffer,
+ nrow: Int,
+ ncol: Int,
+ nonzeros: Int
+ )
+
+ /** With javacpp pointers. */
+ @native def set(@Cast(Array("const void*")) rowJumper: IntPointer,
+ @Cast(Array("const void*")) colIndices: IntPointer,
+ @Const elements: DoublePointer,
+ nrow: Int,
+ ncol: Int,
+ nonzeros: Int
+ )
+
+ @Name(Array("operator="))
+ @native def :=(@Const @ByRef pe: ProdExpression)
+
+ @native def generate_row_block_information()
+
+ /** getters for the compressed_matrix size */
+ //const vcl_size_t & size1() const { return rows_; }
+ @native def size1: Int
+ //const vcl_size_t & size2() const { return cols_; }
+ @native def size2: Int
+ //const vcl_size_t & nnz() const { return nonzeros_; }
+ @native def nnz: Int
+ //const vcl_size_t & blocks1() const { return row_block_num_; }
+ // @native def blocks1: Int
+
+ /** getters for the compressed_matrix buffers */
+ //const handle_type & handle1() const { return row_buffer_; }
+ @native @Const @ByRef def handle1: MemHandle
+ //const handle_type & handle2() const { return col_buffer_; }
+ @native @Const @ByRef def handle2: MemHandle
+ //const handle_type & handle3() const { return row_blocks_; }
+ @native @Const @ByRef def handle3: MemHandle
+ //const handle_type & handle() const { return elements_; }
+ @native @Const @ByRef def handle: MemHandle
+
+}
+
+object CompressedMatrix {
+ Context.loadLib()
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/Context.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/Context.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/Context.scala
new file mode 100644
index 0000000..770f87f
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/Context.scala
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.{Loader, Pointer}
+import org.bytedeco.javacpp.annotation._
+
+/**
+ * This assumes viennacl 1.7.1 is installed, which in ubuntu Xenial defaults to
+ * /usr/include/viennacl, and is installed via
+ * {{{
+ * sudo apt-get install libviennacl-dev
+ * }}}
+ *
+ * @param mtype
+ */
+@Properties(Array(
+ new Platform(
+ includepath = Array("/usr/include/viennacl"),
+ include = Array("matrix.hpp", "compressed_matrix.hpp"),
+ define = Array("VIENNACL_WITH_OPENCL", "VIENNACL_WITH_OPENMP"),
+ compiler = Array("fastfpu","viennacl"),
+ link = Array("OpenCL"),
+ library = "jniViennaCL"
+ )))
+@Namespace("viennacl")
+@Name(Array("context"))
+final class Context(mtype: Int = Context.MEMORY_NOT_INITIALIZED) extends Pointer {
+
+ import Context._
+
+ if (mtype == MEMORY_NOT_INITIALIZED)
+ allocate()
+ else
+ allocate(mtype)
+
+ @native protected def allocate()
+
+ @native protected def allocate(@Cast(Array("viennacl::memory_types")) mtype: Int)
+
+ @Name(Array("memory_type"))
+ @Cast(Array("int"))
+ @native def memoryType: Int
+
+}
+
+object Context {
+
+ def loadLib() = Loader.load(classOf[Context])
+
+ loadLib()
+
+ /* Memory types. Ported from VCL header files. */
+ val MEMORY_NOT_INITIALIZED = 0
+ val MAIN_MEMORY = 1
+ val OPENCL_MEMORY = 2
+ val CUDA_MEMORY = 3
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseColumnMatrix.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseColumnMatrix.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseColumnMatrix.scala
new file mode 100644
index 0000000..7b268e3
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseColumnMatrix.scala
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.{DoublePointer, Pointer}
+import org.bytedeco.javacpp.annotation._
+
+/**
+ * ViennaCL dense matrix, column-major. This is an exact duplication of [[DenseRowMatrix]], and
+ * is only different in the materialized C++ template name. Unfortunately I so far have not figured
+ * out how to handle it with.
+ *
+ * Also, the [[Platform.library]] does not get inherited for some reason, and we really want to
+ * collect all class mappings in the same one libjni.so, so we have to repeat this `library` defi-
+ * nition in every mapped class in this package. (One .so per package convention).
+ */
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform (
+ include=Array("matrix.hpp"),
+ library="jniViennaCL"
+ )))
+@Name(Array("viennacl::matrix<double,viennacl::column_major>"))
+final class DenseColumnMatrix(initDefault:Boolean = true) extends MatrixBase {
+
+ def this(nrow: Int, ncol: Int, ctx: Context = new Context()) {
+ this(false)
+ allocate(nrow, ncol, ctx)
+ }
+
+ def this(data: DoublePointer, nrow: Int, ncol: Int, ctx: Context = new Context(Context.MAIN_MEMORY)) {
+ this(false)
+ allocate(data, ctx.memoryType, nrow, ncol)
+ // We save it to deallocate it ad deallocation time.
+ ptrs += data
+ }
+
+ def this(me: MatMatProdExpression) {
+ this(false)
+ allocate(me)
+ }
+
+ def this(me: MatrixTransExpression) {
+ this(false)
+ allocate(me)
+ }
+
+
+ if (initDefault) allocate()
+
+ @native protected def allocate()
+
+ @native protected def allocate(nrow: Int, ncol: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(data: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))
+ memType: Int,
+ nrow: Int,
+ ncol: Int
+ )
+
+ @native protected def allocate(@Const @ByRef me: MatMatProdExpression)
+
+ @native protected def allocate(@Const @ByRef me: MatrixTransExpression)
+
+}
+
+object DenseColumnMatrix {
+ Context.loadLib()
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseRowMatrix.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseRowMatrix.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseRowMatrix.scala
new file mode 100644
index 0000000..b353924
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/DenseRowMatrix.scala
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.{DoublePointer, Pointer, annotation}
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+/**
+ * ViennaCL dense matrix, row-major.
+ */
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL"
+ )))
+@Name(Array("viennacl::matrix<double,viennacl::row_major>"))
+class DenseRowMatrix(initDefault: Boolean = true) extends MatrixBase {
+
+ def this(nrow: Int, ncol: Int, ctx: Context = new Context()) {
+ this(false)
+ allocate(nrow, ncol, ctx)
+ }
+
+ def this(data: DoublePointer, nrow: Int, ncol: Int, ctx: Context = new Context(Context.MAIN_MEMORY)) {
+ this(false)
+ allocate(data, ctx.memoryType, nrow, ncol)
+ // We save it to deallocate it ad deallocation time.
+ ptrs += data
+ }
+
+ def this(me: MatMatProdExpression) {
+ this(false)
+ allocate(me)
+ }
+
+ def this(me: MatrixTransExpression) {
+ this(false)
+ allocate(me)
+ }
+
+ // TODO: getting compilation errors here
+ def this(sd: SrMatDnMatProdExpression) {
+ this(false)
+ allocate(sd)
+ }
+
+ if (initDefault) allocate()
+
+ @native protected def allocate()
+
+ @native protected def allocate(nrow: Int, ncol: Int, @ByVal ctx: Context)
+
+ @native protected def allocate(data: DoublePointer,
+ @Cast(Array("viennacl::memory_types"))
+ memType: Int,
+ nrow: Int,
+ ncol: Int
+ )
+
+ @native protected def allocate(@Const @ByRef me: MatMatProdExpression)
+
+ @native protected def allocate(@Const @ByRef me: MatrixTransExpression)
+
+ @native protected def allocate(@Const @ByRef me: SrMatDnMatProdExpression)
+
+}
+
+
+object DenseRowMatrix {
+ Context.loadLib()
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatMatProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatMatProdExpression.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatMatProdExpression.scala
new file mode 100644
index 0000000..c88aee5
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatMatProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::matrix_base<double>, " +
+ "const viennacl::matrix_base<double>, " +
+ "viennacl::op_mat_mat_prod>"))
+class MatMatProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatVecProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatVecProdExpression.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatVecProdExpression.scala
new file mode 100644
index 0000000..111cbd3
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatVecProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("vector_expression<const viennacl::matrix_base<double>, " +
+ "const viennacl::vector_base<double>, " +
+ "viennacl::op_prod>"))
+class MatVecProdExpression extends Pointer {
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatrixBase.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatrixBase.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatrixBase.scala
new file mode 100644
index 0000000..6cc1f9f
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MatrixBase.scala
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation._
+
+import scala.collection.mutable.ArrayBuffer
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL"
+ )))
+@Name(Array("viennacl::matrix_base<double>"))
+class MatrixBase extends Pointer {
+
+ protected val ptrs = new ArrayBuffer[Pointer]()
+
+ override def deallocate(deallocate: Boolean): Unit = {
+ super.deallocate(deallocate)
+ ptrs.foreach(_.close())
+ }
+
+ @Name(Array("operator="))
+ @native def :=(@Const @ByRef src: DenseRowMatrix)
+
+ @Name(Array("operator="))
+ @native def :=(@Const @ByRef src: DenseColumnMatrix)
+
+ @Name(Array("size1"))
+ @native
+ def nrow: Int
+
+ @Name(Array("size2"))
+ @native
+ def ncol: Int
+
+ @Name(Array("row_major"))
+ @native
+ def isRowMajor: Boolean
+
+ @Name(Array("internal_size1"))
+ @native
+ def internalnrow: Int
+
+ @Name(Array("internal_size2"))
+ @native
+ def internalncol: Int
+
+ @Name(Array("memory_domain"))
+ @native
+ def memoryDomain: Int
+
+ @Name(Array("switch_memory_context"))
+ @native
+ def switchMemoryContext(@ByRef ctx: Context)
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MemHandle.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MemHandle.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MemHandle.scala
new file mode 100644
index 0000000..73807ac
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/MemHandle.scala
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.{Loader, Pointer}
+import org.bytedeco.javacpp.annotation._
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl::backend")
+@Name(Array("mem_handle"))
+class MemHandle extends Pointer {
+
+ allocate()
+
+ @native def allocate()
+}
+
+object MemHandle {
+
+ def loadLib() = Loader.load(classOf[MemHandle])
+
+ loadLib()
+
+ /* Memory types. Ported from VCL header files. */
+ val MEMORY_NOT_INITIALIZED = 0
+ val MAIN_MEMORY = 1
+ val OPENCL_MEMORY = 2
+ val CUDA_MEMORY = 3
+
+}
http://git-wip-us.apache.org/repos/asf/mahout/blob/034790cc/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/ProdExpression.scala
----------------------------------------------------------------------
diff --git a/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/ProdExpression.scala b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/ProdExpression.scala
new file mode 100644
index 0000000..7ee42b8
--- /dev/null
+++ b/viennacl/src/main/scala/org/apache/mahout/viennacl/opencl/javacpp/ProdExpression.scala
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.viennacl.opencl.javacpp
+
+import org.bytedeco.javacpp.Pointer
+import org.bytedeco.javacpp.annotation.{Name, Namespace, Platform, Properties}
+
+
+@Properties(inherit = Array(classOf[Context]),
+ value = Array(new Platform(
+ library = "jniViennaCL")
+ ))
+@Namespace("viennacl")
+@Name(Array("matrix_expression<const viennacl::compressed_matrix<double>, " +
+ "const viennacl::compressed_matrix<double>, " +
+ "viennacl::op_prod>"))
+class ProdExpression extends Pointer {
+
+}