You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by dl...@apache.org on 2014/07/22 00:00:40 UTC
git commit: MAHOUT-1529(e): Move dense/sparse matrix test in mapBlock
into spark (Anand Avati via dlyubimov) All math-scala,
spark tests are passing. this closes apache/mahout#29
Repository: mahout
Updated Branches:
refs/heads/master 272b16153 -> dec441fb8
MAHOUT-1529(e): Move dense/sparse matrix test in mapBlock into spark (Anand Avati via dlyubimov)
All math-scala, spark tests are passing.
this closes apache/mahout#29
Squashed commit of the following:
commit 1e3cdb68198636c9f38f2d41d782d12edba7a2f7
Author: Anand Avati <av...@redhat.com>
Date: Mon Jul 14 17:20:09 2014 -0700
MAHOUT-1529: Move dense/sparse matrix test in mapBlock into spark/
In h2o engine, the Matrix provided to mapBlock() is an instance of
"H2OBlockMatrix extends AbstractMatrix", and neither a DenseMatrix
nor SparseMatrix. H2OBlockMatrix is a 0-copy virtual Matrix exposing
just the partition's data (created at almost no expense), and creates
a copy-on-write Matrix only if modified by the blockmapfunction.
So these two tests are failing with h2obindings. Hence moving these two
tests into spark module.
Signed-off-by: Anand Avati <av...@redhat.com>
Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/dec441fb
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/dec441fb
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/dec441fb
Branch: refs/heads/master
Commit: dec441fb895c96d1e756619d15d75bba00b10fa3
Parents: 272b161
Author: Dmitriy Lyubimov <dl...@apache.org>
Authored: Mon Jul 21 14:54:57 2014 -0700
Committer: Dmitriy Lyubimov <dl...@apache.org>
Committed: Mon Jul 21 14:58:10 2014 -0700
----------------------------------------------------------------------
CHANGELOG | 4 ++-
.../mahout/math/drm/DrmLikeSuiteBase.scala | 29 ------------------
.../mahout/sparkbindings/drm/DrmLikeSuite.scala | 31 +++++++++++++++++++-
3 files changed, 33 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mahout/blob/dec441fb/CHANGELOG
----------------------------------------------------------------------
diff --git a/CHANGELOG b/CHANGELOG
index 7ea84c5..f34ee13 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -2,7 +2,9 @@ Mahout Change Log
Release 1.0 - unreleased
- MAHOUT-1583: cbind() operator for Scala DRMs
+ MAHOUT-1529(e): Move dense/sparse matrix test in mapBlock into spark (Anand Avati via dlyubimov)
+
+ MAHOUT-1583: cbind() operator for Scala DRMs (dlyubimov)
MAHOUT-1541, MAHOUT-1568, MAHOUT-1569: Created text-delimited file I/O traits and classes on spark, a MahoutDriver for a CLI and a ItemSimilairtyDriver using the CLI
http://git-wip-us.apache.org/repos/asf/mahout/blob/dec441fb/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala
----------------------------------------------------------------------
diff --git a/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala b/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala
index 651c611..eaca53f 100644
--- a/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala
+++ b/math-scala/src/test/scala/org/apache/mahout/math/drm/DrmLikeSuiteBase.scala
@@ -50,35 +50,6 @@ trait DrmLikeSuiteBase extends DistributedMahoutSuite with Matchers {
}
- test("DRM blockify dense") {
-
- val inCoreA = dense((1, 2, 3), (3, 4, 5))
- val drmA = drmParallelize(inCoreA, numPartitions = 2)
-
- (inCoreA - drmA.mapBlock() {
- case (keys, block) =>
- if (!block.isInstanceOf[DenseMatrix])
- throw new AssertionError("Block must be dense.")
- keys -> block
- }).norm should be < 1e-4
- }
-
- test("DRM blockify sparse -> SRM") {
-
- val inCoreA = sparse(
- (1, 2, 3),
- 0 -> 3 :: 2 -> 5 :: Nil
- )
- val drmA = drmParallelize(inCoreA, numPartitions = 2)
-
- (inCoreA - drmA.mapBlock() {
- case (keys, block) =>
- if (!block.isInstanceOf[SparseRowMatrix])
- throw new AssertionError("Block must be dense.")
- keys -> block
- }).norm should be < 1e-4
- }
-
test("DRM parallelizeEmpty") {
val drmEmpty = drmParallelizeEmpty(100, 50)
http://git-wip-us.apache.org/repos/asf/mahout/blob/dec441fb/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala b/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala
index e6a9055..bf635dc 100644
--- a/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala
+++ b/spark/src/test/scala/org/apache/mahout/sparkbindings/drm/DrmLikeSuite.scala
@@ -27,4 +27,33 @@ import org.apache.mahout.sparkbindings.test.DistributedSparkSuite
/** DRMLike tests -- just run common DRM tests in Spark. */
-class DrmLikeSuite extends FunSuite with DistributedSparkSuite with DrmLikeSuiteBase
+class DrmLikeSuite extends FunSuite with DistributedSparkSuite with DrmLikeSuiteBase {
+ test("DRM blockify dense") {
+
+ val inCoreA = dense((1, 2, 3), (3, 4, 5))
+ val drmA = drmParallelize(inCoreA, numPartitions = 2)
+
+ (inCoreA - drmA.mapBlock() {
+ case (keys, block) =>
+ if (!block.isInstanceOf[DenseMatrix])
+ throw new AssertionError("Block must be dense.")
+ keys -> block
+ }).norm should be < 1e-4
+ }
+
+ test("DRM blockify sparse -> SRM") {
+
+ val inCoreA = sparse(
+ (1, 2, 3),
+ 0 -> 3 :: 2 -> 5 :: Nil
+ )
+ val drmA = drmParallelize(inCoreA, numPartitions = 2)
+
+ (inCoreA - drmA.mapBlock() {
+ case (keys, block) =>
+ if (!block.isInstanceOf[SparseRowMatrix])
+ throw new AssertionError("Block must be dense.")
+ keys -> block
+ }).norm should be < 1e-4
+ }
+}