You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by pa...@apache.org on 2014/06/05 03:41:09 UTC

[1/5] git commit: added Sebastian's CooccurrenceAnalysis patch updated it to use current Mahout-DSL

Repository: mahout
Updated Branches:
  refs/heads/mahout-1541 [created] 8a4b4347d


added Sebastian's CooccurrenceAnalysis patch updated it to use current Mahout-DSL


Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/107a0ba9
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/107a0ba9
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/107a0ba9

Branch: refs/heads/mahout-1541
Commit: 107a0ba9605241653a85b113661a8fa5c055529f
Parents: 127bd01
Author: pferrel <pa...@occamsmachete.com>
Authored: Wed Jun 4 12:54:22 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Wed Jun 4 12:54:22 2014 -0700

----------------------------------------------------------------------
 .gitignore                                      |   1 +
 spark/src/.DS_Store                             | Bin 0 -> 6148 bytes
 spark/src/main/.DS_Store                        | Bin 0 -> 6148 bytes
 spark/src/main/scala/.DS_Store                  | Bin 0 -> 6148 bytes
 spark/src/main/scala/org/.DS_Store              | Bin 0 -> 6148 bytes
 spark/src/main/scala/org/apache/.DS_Store       | Bin 0 -> 6148 bytes
 .../apache/mahout/cf/CooccurrenceAnalysis.scala | 210 +++++++++++++++++++
 .../mahout/cf/examples/Recommendations.scala    | 169 +++++++++++++++
 8 files changed, 380 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index c47bff1..f500375 100644
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,7 @@ output-asf-email-examples/
 .project
 .settings/
 .idea/
+.DS_Store
 *.iml
 target/
 examples/bin/tmp

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/.DS_Store
----------------------------------------------------------------------
diff --git a/spark/src/.DS_Store b/spark/src/.DS_Store
new file mode 100644
index 0000000..7b0d367
Binary files /dev/null and b/spark/src/.DS_Store differ

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/main/.DS_Store
----------------------------------------------------------------------
diff --git a/spark/src/main/.DS_Store b/spark/src/main/.DS_Store
new file mode 100644
index 0000000..7ac63ad
Binary files /dev/null and b/spark/src/main/.DS_Store differ

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/main/scala/.DS_Store
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/.DS_Store b/spark/src/main/scala/.DS_Store
new file mode 100644
index 0000000..e6dc460
Binary files /dev/null and b/spark/src/main/scala/.DS_Store differ

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/main/scala/org/.DS_Store
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/.DS_Store b/spark/src/main/scala/org/.DS_Store
new file mode 100644
index 0000000..d6999d3
Binary files /dev/null and b/spark/src/main/scala/org/.DS_Store differ

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/main/scala/org/apache/.DS_Store
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/.DS_Store b/spark/src/main/scala/org/apache/.DS_Store
new file mode 100644
index 0000000..4ba4fae
Binary files /dev/null and b/spark/src/main/scala/org/apache/.DS_Store differ

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/main/scala/org/apache/mahout/cf/CooccurrenceAnalysis.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/cf/CooccurrenceAnalysis.scala b/spark/src/main/scala/org/apache/mahout/cf/CooccurrenceAnalysis.scala
new file mode 100644
index 0000000..5df329b
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/cf/CooccurrenceAnalysis.scala
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.cf
+
+import org.apache.mahout.math._
+import scalabindings._
+import RLikeOps._
+import drm._
+import RLikeDrmOps._
+import org.apache.mahout.sparkbindings._
+
+import scala.collection.JavaConversions._
+import org.apache.mahout.math.stats.LogLikelihood
+import collection._
+// import scala.collection.parallel.mutable
+import org.apache.mahout.common.RandomUtils
+
+
+/**
+ * based on "Ted Dunnning & Ellen Friedman: Practical Machine Learning, Innovations in Recommendation",
+ * available at http://www.mapr.com/practical-machine-learning
+ *
+ * see also "Sebastian Schelter, Christoph Boden, Volker Markl:
+ * Scalable Similarity-Based Neighborhood Methods with MapReduce
+ * ACM Conference on Recommender Systems 2012"
+ */
+object CooccurrenceAnalysis extends Serializable {
+
+  /** Compares (Int,Double) pairs by the second value */
+  private val orderByScore = Ordering.fromLessThan[(Int, Double)] { case ((_, score1), (_, score2)) => score1 > score2 }
+
+  def cooccurrences(drmARaw: DrmLike[Int], randomSeed: Int = 0xdeadbeef, maxInterestingItemsPerThing: Int = 50,
+                    maxNumInteractions: Int = 500, drmBs: Array[DrmLike[Int]] = Array()): List[DrmLike[Int]] = {
+
+    implicit val disributedContext = drmARaw.context
+
+    // Apply selective downsampling, pin resulting matrix
+    val drmA = sampleDownAndBinarize(drmARaw, randomSeed, maxNumInteractions).checkpoint()
+
+    // num users, which equals the maximum number of interactions per item
+    val numUsers = drmA.nrow.toInt
+
+    // Compute & broadcast the number of interactions per thing in A
+    val bcastInteractionsPerItemA = drmBroadcast(drmA.colSums)
+
+    // Compute co-occurrence matrix A'A
+    val drmAtA = drmA.t %*% drmA
+
+    // Compute loglikelihood scores and sparsify the resulting matrix to get the indicator matrix
+    val drmIndicatorsAtA = computeIndicators(drmAtA, numUsers, maxInterestingItemsPerThing, bcastInteractionsPerItemA,
+                                             bcastInteractionsPerItemA, crossCooccurrence = false)
+
+    var indicatorMatrices = List(drmIndicatorsAtA)
+
+    // Now look at cross-co-occurrences
+    for (drmBRaw <- drmBs) {
+      // Down-sample and pin other interaction matrix
+      val drmB = sampleDownAndBinarize(drmBRaw, randomSeed, maxNumInteractions).checkpoint()
+
+      // Compute & broadcast the number of interactions per thing in B
+      val bcastInteractionsPerThingB = drmBroadcast(drmB.colSums)
+
+      // Compute cross-co-occurrence matrix B'A
+      val drmBtA = drmB.t %*% drmA
+      
+      val drmIndicatorsBtA = computeIndicators(drmBtA, numUsers, maxInterestingItemsPerThing,
+                                               bcastInteractionsPerThingB, bcastInteractionsPerItemA)
+
+      indicatorMatrices = indicatorMatrices :+ drmIndicatorsBtA
+
+      drmB.uncache()
+    }
+
+    // Unpin downsampled interaction matrix
+    drmA.uncache()
+
+    // Return list of indicator matrices
+    indicatorMatrices
+  }
+
+  /**
+   * Compute loglikelihood ratio
+   * see http://tdunning.blogspot.de/2008/03/surprise-and-coincidence.html for details
+   **/
+  def loglikelihoodRatio(numInteractionsWithA: Long, numInteractionsWithB: Long,
+                         numInteractionsWithAandB: Long, numInteractions: Long) = {
+
+    val k11 = numInteractionsWithAandB
+    val k12 = numInteractionsWithA - numInteractionsWithAandB
+    val k21 = numInteractionsWithB - numInteractionsWithAandB
+    val k22 = numInteractions - numInteractionsWithA - numInteractionsWithB + numInteractionsWithAandB
+
+    LogLikelihood.logLikelihoodRatio(k11, k12, k21, k22)
+  }
+
+  def computeIndicators(drmBtA: DrmLike[Int], numUsers: Int, maxInterestingItemsPerThing: Int,
+      bcastNumInteractionsB: BCast[Vector], bcastNumInteractionsA: BCast[Vector],
+      crossCooccurrence: Boolean = true) = {
+    drmBtA.mapBlock() {
+      case (keys, block) =>
+
+        val llrBlock = block.like()
+        val numInteractionsB: Vector = bcastNumInteractionsB
+        val numInteractionsA: Vector = bcastNumInteractionsA
+
+        for (index <- 0 until keys.size) {
+
+          val thingB = keys(index)
+
+          // PriorityQueue to select the top-k items
+          val topItemsPerThing = new mutable.PriorityQueue[(Int,Double)]()(orderByScore)
+
+          block(index, ::).nonZeroes().foreach { elem =>
+            val thingA = elem.index
+            val cooccurrences = elem.get
+
+            // exclude co-occurrences of the item with itself
+            if (crossCooccurrence || thingB != thingA) {
+              // Compute loglikelihood ratio
+              val llrRatio = loglikelihoodRatio(numInteractionsB(thingB).toLong, numInteractionsA(thingA).toLong,
+                                                cooccurrences.toLong, numUsers)
+              val candidate = thingA -> llrRatio
+
+              // Enqueue item with score, if belonging to the top-k
+              if (topItemsPerThing.size < maxInterestingItemsPerThing) {
+                topItemsPerThing.enqueue(candidate)
+              } else if (orderByScore.lt(candidate, topItemsPerThing.head)) {
+                topItemsPerThing.dequeue()
+                topItemsPerThing.enqueue(candidate)
+              }
+            }
+          }
+
+          // Add top-k interesting items to the output matrix
+          topItemsPerThing.dequeueAll.foreach { case (otherThing, llrScore) => llrBlock(index, otherThing) = llrScore }
+        }
+
+        keys -> llrBlock
+    }
+  }
+
+  /**
+   * Selectively downsample users and things with an anormalous amount of interactions, inspired by
+   * https://github.com/tdunning/in-memory-cooccurrence/blob/master/src/main/java/com/tdunning/cooc/Analyze.java
+   *
+   * additionally binarizes input matrix, as we're only interesting in knowing whether interactions happened or not
+   */
+  def sampleDownAndBinarize(drmM: DrmLike[Int], seed: Int, maxNumInteractions: Int) = {
+
+    implicit val distributedContext = drmM.context
+
+    // Pin raw interaction matrix
+    val drmI = drmM.checkpoint()
+
+    // Broadcast vector containing the number of interactions with each thing
+    val bcastNumInteractions = drmBroadcast(drmI.colSums)
+
+    val downSampledDrmI = drmI.mapBlock() {
+      case (keys, block) =>
+        val numInteractions: Vector = bcastNumInteractions
+
+        // Use a hash of the unique first key to seed the RNG, makes this computation repeatable in case of failures
+        val random = RandomUtils.getRandom(MurmurHash.hash(keys(0), seed))
+
+        val downsampledBlock = block.like()
+
+        // Downsample the interaction vector of each user
+        for (userIndex <- 0 until keys.size) {
+
+          val interactionsOfUser = block(userIndex, ::)
+          val numInteractionsOfUser = interactionsOfUser.sum
+
+          val perUserSampleRate = math.min(maxNumInteractions, numInteractionsOfUser) / numInteractionsOfUser
+
+          interactionsOfUser.nonZeroes().foreach { elem =>
+            val numInteractionsWithThing = numInteractions(elem.index)
+            val perThingSampleRate = math.min(maxNumInteractions, numInteractionsWithThing) / numInteractionsWithThing
+
+            if (random.nextDouble() <= math.min(perUserSampleRate, perThingSampleRate)) {
+              // We ignore the original interaction value and create a binary 0-1 matrix
+              // as we only consider whether interactions happened or did not happen
+              downsampledBlock(userIndex, elem.index) = 1
+            }
+          }
+      }
+
+      keys -> downsampledBlock
+    }
+
+    // Unpin raw interaction matrix
+    drmI.uncache()
+
+    downSampledDrmI
+  }
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/107a0ba9/spark/src/main/scala/org/apache/mahout/cf/examples/Recommendations.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/cf/examples/Recommendations.scala b/spark/src/main/scala/org/apache/mahout/cf/examples/Recommendations.scala
new file mode 100644
index 0000000..c640e1e
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/cf/examples/Recommendations.scala
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.cf.examples
+
+import scala.io.Source
+import org.apache.mahout.math._
+import scalabindings._
+import RLikeOps._
+import drm._
+import RLikeDrmOps._
+import org.apache.mahout.sparkbindings._
+
+import org.apache.mahout.cf.CooccurrenceAnalysis._
+import scala.collection.JavaConversions._
+
+/**
+ * The Epinions dataset contains ratings from users to items and a trust-network between the users.
+ * We use co-occurrence analysis to compute "users who like these items, also like that items" and
+ * "users who trust these users, like that items"
+ *
+ * Download and unpack the dataset files from:
+ *
+ * http://www.trustlet.org/datasets/downloaded_epinions/ratings_data.txt.bz2
+ * http://www.trustlet.org/datasets/downloaded_epinions/trust_data.txt.bz2
+ **/
+object RunCrossCooccurrenceAnalysisOnEpinions {
+
+  def main(args: Array[String]): Unit = {
+
+    if (args.length == 0) {
+      println("Usage: RunCooccurrenceAnalysisOnMovielens1M <path-to-dataset-folder>")
+      println("Download the dataset from http://www.trustlet.org/datasets/downloaded_epinions/ratings_data.txt.bz2 and")
+      println("http://www.trustlet.org/datasets/downloaded_epinions/trust_data.txt.bz2")
+      sys.exit(-1)
+    }
+
+    val datasetDir = args(0)
+
+    val epinionsRatings = new SparseMatrix(49290, 139738)
+
+    var firstLineSkipped = false
+    for (line <- Source.fromFile(datasetDir + "/ratings_data.txt").getLines()) {
+      if (line.contains(' ') && firstLineSkipped) {
+        val tokens = line.split(' ')
+        val userID = tokens(0).toInt - 1
+        val itemID = tokens(1).toInt - 1
+        val rating = tokens(2).toDouble
+        epinionsRatings(userID, itemID) = rating
+      }
+      firstLineSkipped = true
+    }
+
+    val epinionsTrustNetwork = new SparseMatrix(49290, 49290)
+    firstLineSkipped = false
+    for (line <- Source.fromFile(datasetDir + "/trust_data.txt").getLines()) {
+      if (line.contains(' ') && firstLineSkipped) {
+        val tokens = line.trim.split(' ')
+        val userID = tokens(0).toInt - 1
+        val trustedUserId = tokens(1).toInt - 1
+        epinionsTrustNetwork(userID, trustedUserId) = 1
+      }
+      firstLineSkipped = true
+    }
+
+    System.setProperty("spark.kryo.referenceTracking", "false")
+    System.setProperty("spark.kryoserializer.buffer.mb", "100")
+
+//    implicit val distributedContext = mahoutSparkContext(masterUrl = "local", appName = "MahoutLocalContext",
+//      customJars = Traversable.empty[String])
+    implicit val distributedContext = mahoutSparkContext(masterUrl = "spark://occam4:7077", appName = "MahoutClusteredContext",
+      customJars = Traversable.empty[String])
+
+    val drmEpinionsRatings = drmParallelize(epinionsRatings, numPartitions = 2)
+    val drmEpinionsTrustNetwork = drmParallelize(epinionsTrustNetwork, numPartitions = 2)
+
+    val indicatorMatrices = cooccurrences(drmEpinionsRatings, randomSeed = 0xdeadbeef,
+        maxInterestingItemsPerThing = 100, maxNumInteractions = 500, Array(drmEpinionsTrustNetwork))
+//hdfs://occam4:54310/user/pat/xrsj/
+/*
+    RecommendationExamplesHelper.saveIndicatorMatrix(indicatorMatrices(0),
+        "/tmp/co-occurrence-on-epinions/indicators-item-item/")
+    RecommendationExamplesHelper.saveIndicatorMatrix(indicatorMatrices(1),
+        "/tmp/co-occurrence-on-epinions/indicators-trust-item/")
+*/
+    RecommendationExamplesHelper.saveIndicatorMatrix(indicatorMatrices(0),
+      "hdfs://occam4:54310/user/pat/xrsj/indicators-item-item/")
+    RecommendationExamplesHelper.saveIndicatorMatrix(indicatorMatrices(1),
+      "hdfs://occam4:54310/user/pat/xrsj/indicators-trust-item/")
+
+    distributedContext.close()
+
+    println("Saved indicators to /tmp/co-occurrence-on-epinions/")
+  }
+}
+
+/**
+ * The movielens1M dataset contains movie ratings, we use co-occurrence analysis to compute
+ * "users who like these movies, also like that movies"
+ *
+ * Download and unpack the dataset files from:
+ * http://files.grouplens.org/datasets/movielens/ml-1m.zip
+ */
+object RunCooccurrenceAnalysisOnMovielens1M {
+
+  def main(args: Array[String]): Unit = {
+
+    if (args.length == 0) {
+      println("Usage: RunCooccurrenceAnalysisOnMovielens1M <path-to-dataset-folder>")
+      println("Download the dataset from http://files.grouplens.org/datasets/movielens/ml-1m.zip")
+      sys.exit(-1)
+    }
+
+    val datasetDir = args(0)
+
+    System.setProperty("spark.kryo.referenceTracking", "false")
+    System.setProperty("spark.kryoserializer.buffer.mb", "100")
+
+    implicit val sc = mahoutSparkContext(masterUrl = "local", appName = "MahoutLocalContext",
+      customJars = Traversable.empty[String])
+
+    System.setProperty("mahout.math.AtA.maxInMemNCol", 4000.toString)
+
+    val movielens = new SparseMatrix(6040, 3952)
+
+    for (line <- Source.fromFile(datasetDir + "/ratings.dat").getLines()) {
+      val tokens = line.split("::")
+      val userID = tokens(0).toInt - 1
+      val itemID = tokens(1).toInt - 1
+      val rating = tokens(2).toDouble
+      movielens(userID, itemID) = rating
+    }
+
+    val drmMovielens = drmParallelize(movielens, numPartitions = 2)
+
+    val indicatorMatrix = cooccurrences(drmMovielens).head
+
+    RecommendationExamplesHelper.saveIndicatorMatrix(indicatorMatrix,
+        "/tmp/co-occurrence-on-movielens/indicators-item-item/")
+
+    sc.stop()
+
+    println("Saved indicators to /tmp/co-occurrence-on-movielens/")
+  }
+}
+
+object RecommendationExamplesHelper {
+
+  def saveIndicatorMatrix(indicatorMatrix: DrmLike[Int], path: String) = {
+    indicatorMatrix.rdd.flatMap({ case (thingID, itemVector) =>
+        for (elem <- itemVector.nonZeroes()) yield { thingID + '\t' + elem.index }
+      })
+      .saveAsTextFile(path)
+  }
+}


[4/5] git commit: added Sebastian's MurmurHash changes

Posted by pa...@apache.org.
added Sebastian's MurmurHash changes

Signed-off-by: pferrel <pa...@occamsmachete.com>


Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/16c03f7f
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/16c03f7f
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/16c03f7f

Branch: refs/heads/mahout-1541
Commit: 16c03f7fa73c156859d1dba3a333ef9e8bf922b0
Parents: 107a0ba
Author: pferrel <pa...@occamsmachete.com>
Authored: Wed Jun 4 14:32:18 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Wed Jun 4 14:32:18 2014 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/mahout/math/MurmurHash.java   | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mahout/blob/16c03f7f/math/src/main/java/org/apache/mahout/math/MurmurHash.java
----------------------------------------------------------------------
diff --git a/math/src/main/java/org/apache/mahout/math/MurmurHash.java b/math/src/main/java/org/apache/mahout/math/MurmurHash.java
index 0b3fab0..32dfdd6 100644
--- a/math/src/main/java/org/apache/mahout/math/MurmurHash.java
+++ b/math/src/main/java/org/apache/mahout/math/MurmurHash.java
@@ -17,6 +17,8 @@
 
 package org.apache.mahout.math;
 
+import com.google.common.primitives.Ints;
+
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
 
@@ -29,7 +31,16 @@ import java.nio.ByteOrder;
  */
 public final class MurmurHash {
 
-  private MurmurHash() {
+  private MurmurHash() {}
+
+  /**
+   * Hashes an int.
+   * @param data The int to hash.
+   * @param seed The seed for the hash.
+   * @return The 32 bit hash of the bytes in question.
+   */
+  public static int hash(int data, int seed) {
+    return hash(ByteBuffer.wrap(Ints.toByteArray(data)), seed);
   }
 
   /**


[5/5] git commit: Merge branch 'mahout-1464' into mahout-1541

Posted by pa...@apache.org.
Merge branch 'mahout-1464' into mahout-1541


Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/8a4b4347
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/8a4b4347
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/8a4b4347

Branch: refs/heads/mahout-1541
Commit: 8a4b4347ddb7b9ac97590aa20189d89d8a07a80a
Parents: a592659 16c03f7
Author: pferrel <pa...@occamsmachete.com>
Authored: Wed Jun 4 14:33:11 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Wed Jun 4 14:33:11 2014 -0700

----------------------------------------------------------------------
 .../main/java/org/apache/mahout/math/MurmurHash.java   | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------



[3/5] git commit: added scopt to pom deps

Posted by pa...@apache.org.
added scopt to pom deps


Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/a5926593
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/a5926593
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/a5926593

Branch: refs/heads/mahout-1541
Commit: a59265931ed3a51ba81e1a0a7171ebb102be4fa4
Parents: 74b9921
Author: pferrel <pa...@occamsmachete.com>
Authored: Wed Jun 4 13:13:13 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Wed Jun 4 13:13:13 2014 -0700

----------------------------------------------------------------------
 spark/pom.xml | 7 +++++++
 1 file changed, 7 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mahout/blob/a5926593/spark/pom.xml
----------------------------------------------------------------------
diff --git a/spark/pom.xml b/spark/pom.xml
index ac99ffd..1f08d9a 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -288,6 +288,13 @@
 
 
     <!--  3rd-party -->
+
+    <dependency>
+      <groupId>com.github.scopt</groupId>
+      <artifactId>scopt_2.10</artifactId>
+      <version>3.2.0</version>
+    </dependency>
+
     <!-- spark stuff -->
     <dependency>
       <groupId>org.apache.spark</groupId>


[2/5] git commit: Adding stuff for itemsimilarity driver for Spark

Posted by pa...@apache.org.
Adding stuff for itemsimilarity driver for Spark


Project: http://git-wip-us.apache.org/repos/asf/mahout/repo
Commit: http://git-wip-us.apache.org/repos/asf/mahout/commit/74b9921c
Tree: http://git-wip-us.apache.org/repos/asf/mahout/tree/74b9921c
Diff: http://git-wip-us.apache.org/repos/asf/mahout/diff/74b9921c

Branch: refs/heads/mahout-1541
Commit: 74b9921c4c9bd8903585bbd74d9e66298ea8b7a0
Parents: 107a0ba
Author: pferrel <pa...@occamsmachete.com>
Authored: Wed Jun 4 13:09:07 2014 -0700
Committer: pferrel <pa...@occamsmachete.com>
Committed: Wed Jun 4 13:09:07 2014 -0700

----------------------------------------------------------------------
 spark/src/.DS_Store                             | Bin 6148 -> 0 bytes
 .../apache/mahout/drivers/FileSysUtils.scala    |  34 +++
 .../apache/mahout/drivers/IndexedDataset.scala  |  54 +++++
 .../drivers/IndexedDatasetDriverTest.scala      | 164 +++++++++++++++
 .../mahout/drivers/ItemSimilarityDriver.scala   | 208 +++++++++++++++++++
 .../apache/mahout/drivers/MahoutDriver.scala    |  74 +++++++
 .../mahout/drivers/MahoutOptionParser.scala     |  24 +++
 .../apache/mahout/drivers/ReaderWriter.scala    | 188 +++++++++++++++++
 .../org/apache/mahout/drivers/Schema.scala      |  30 +++
 .../mahout/drivers/IndexedDatasetTest.scala     |  25 +++
 .../drivers/ItemSimilarityDriver$Test.scala     | 180 ++++++++++++++++
 11 files changed, 981 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/.DS_Store
----------------------------------------------------------------------
diff --git a/spark/src/.DS_Store b/spark/src/.DS_Store
deleted file mode 100644
index 7b0d367..0000000
Binary files a/spark/src/.DS_Store and /dev/null differ

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/FileSysUtils.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/FileSysUtils.scala b/spark/src/main/scala/org/apache/mahout/drivers/FileSysUtils.scala
new file mode 100644
index 0000000..e8491f2
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/FileSysUtils.scala
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.drivers
+
+/**
+  * Returns a [[java.lang.String]]comma delimited list of URIs discovered based on parameters in the constructor.
+  * The String is formatted to be input into [[org.apache.spark.SparkContext.textFile()]]
+  *
+  * @param pathURI Where to start looking for inFiles, only HDFS and local URI are currently
+  *                supported
+  * @param filePattern regex that must match the entire filename to have the file included in the returned list
+  * @param recursive true traverses the filesystem recursively
+  */
+
+case class FileSysUtils(pathURI: String, filePattern: String = "", recursive: Boolean = false) {
+  // todo: There is an HDFS filestatus method that collects multiple inFiles, see if this is the right thing to use
+  // todo: check to see if the input is a supported URI for collection or recursive search but just pass through otherwise
+  def uris = {pathURI}
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/IndexedDataset.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/IndexedDataset.scala b/spark/src/main/scala/org/apache/mahout/drivers/IndexedDataset.scala
new file mode 100644
index 0000000..36bda90
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/IndexedDataset.scala
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.drivers
+
+import com.google.common.collect.BiMap
+import org.apache.mahout.math.drm.DrmLike
+
+/**
+  * Wraps a [[org.apache.mahout.sparkbindings.drm.DrmLike]] object with two [[com.google.common.collect.BiMap]]s to store ID/label translation dictionaries.
+  * The purpose of this class is to wrap a DrmLike[C] with bidirectional ID mappings so
+  * a user specified label or ID can be stored and mapped to and from the [[scala.Int]] ordinal ID
+  * used internal to Mahout Core code.
+  *
+  * Example: For a transpose job the [[org.apache.mahout.drivers.IndexedDataset#matrix]]: [[org.apache.mahout.sparkbindings.drm.DrmLike]] is passed into the DSL code
+  * that transposes the values, then a resulting [[org.apache.mahout.drivers.IndexedDataset]] is created from the transposed DrmLike object with swapped dictionaries (since the rows and columns are transposed). The new
+  * [[org.apache.mahout.drivers.IndexedDataset]] is returned.
+  *
+  * @param matrix  DrmLike[Int], representing the distributed matrix storing the actual data.
+  * @param rowIDs BiMap[String, Int] storing a bidirectional mapping of external String ID to
+  *                  and from the ordinal Mahout Int ID. This one holds row labels
+  * @param columnIDs BiMap[String, Int] storing a bidirectional mapping of external String
+  *                  ID to and from the ordinal Mahout Int ID. This one holds column labels
+  */
+
+case class IndexedDataset(matrix: DrmLike[Int], rowIDs: BiMap[String,Int], columnIDs: BiMap[String,Int]) {
+}
+
+/**
+  * Companion object for the case class [[org.apache.mahout.drivers.IndexedDataset]] primarily used to get a secondary constructor for
+  * making one [[org.apache.mahout.drivers.IndexedDataset]] from another. Used when you have a factory like [[org.apache.mahout.drivers.IndexedDatasetStore]]
+  * {{{
+  * val indexedDataset = IndexedDataset(indexedDatasetStore.read)
+  * }}}
+  */
+
+object IndexedDataset {
+  /** Secondary constructor for [[org.apache.mahout.drivers.IndexedDataset]] */
+  def apply(id2: IndexedDataset) = new IndexedDataset(id2.matrix,  id2.rowIDs, id2.columnIDs)
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/IndexedDatasetDriverTest.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/IndexedDatasetDriverTest.scala b/spark/src/main/scala/org/apache/mahout/drivers/IndexedDatasetDriverTest.scala
new file mode 100644
index 0000000..80fc134
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/IndexedDatasetDriverTest.scala
@@ -0,0 +1,164 @@
+package org.apache.mahout.drivers
+
+import org.apache.mahout.sparkbindings._
+import java.io.{FileWriter, BufferedWriter}
+import com.google.common.io.Closeables
+
+object ItemSimilarityDriverTest {
+
+  def main(args: Array[String]): Unit = {
+
+    System.setProperty("spark.kryo.referenceTracking", "false")
+    System.setProperty("spark.kryoserializer.buffer.mb", "100")
+
+    implicit val sc = mahoutSparkContext(masterUrl = "local", appName = "MahoutLocalContext",
+      customJars = Traversable.empty[String])
+
+    val exampleCsvlogStatements = Array(
+      "12569537329,user1,item1,\"view\"",
+      "12569537329,user1,item2,\"view\"",
+      "12569537329,user1,item2,\"like\"",
+      "12569537329,user1,item3,\"like\"",
+      "12569537329,user2,item2,\"view\"",
+      "12569537329,user2,item2,\"like\"",
+      "12569537329,user3,item1,\"like\"",
+      "12569537329,user3,item3,\"view\"",
+      "12569537329,user3,item3,\"like\""
+    )
+
+    val exampleTsvLogStatements = Array(
+      "12569537329\tuser1\titem1\t\"view\"",
+      "12569537329\tuser1\titem2\t\"view\"",
+      "12569537329\tuser1\titem2\t\"like\"",
+      "12569537329\tuser1\titem3\t\"like\"",
+      "12569537329\tuser2\titem2\t\"view\"",
+      "12569537329\tuser2\titem2\t\"like\"",
+      "12569537329\tuser3\titem1\t\"like\"",
+      "12569537329\tuser3\titem3\t\"view\"",
+      "12569537329\tuser3\titem3\t\"like\""
+    )
+
+
+    val csvLogStatements = Array(
+      "u1,purchase,iphone",
+      "u1,purchase,ipad",
+      "u2,purchase,nexus",
+      "u2,purchase,galaxy",
+      "u3,purchase,surface",
+      "u4,purchase,iphone",
+      "u4,purchase,galaxy",
+      "u1,view,iphone",
+      "u1,view,ipad",
+      "u1,view,nexus",
+      "u1,view,galaxy",
+      "u2,view,iphone",
+      "u2,view,ipad",
+      "u2,view,nexus",
+      "u2,view,galaxy",
+      "u3,view,surface",
+      "u3,view,nexus",
+      "u4,view,iphone",
+      "u4,view,ipad",
+      "u4,view,galaxy"
+    )
+
+    val tsvLogStatements = Array(
+      "u1\tpurchase\tiphone",
+      "u1\tpurchase\tipad",
+      "u2\tpurchase\tnexus",
+      "u2\tpurchase\tgalaxy",
+      "u3\tpurchase\tsurface",
+      "u4\tpurchase\tiphone",
+      "u4\tpurchase\tgalaxy",
+      "u1\tview\tiphone",
+      "u1\tview\tipad",
+      "u1\tview\tnexus",
+      "u1\tview\tgalaxy",
+      "u2\tview\tiphone",
+      "u2\tview\tipad",
+      "u2\tview\tnexus",
+      "u2\tview\tgalaxy",
+      "u3\tview\tsurface",
+      "u3\tview\tnexus",
+      "u4\tview\tiphone",
+      "u4\tview\tipad",
+      "u4\tview\tgalaxy"
+    )
+
+    var w: BufferedWriter = null
+    try {
+      w = new BufferedWriter(new FileWriter("tmp/cf-data.txt"))
+      w.write(csvLogStatements.mkString("\n"))
+    } finally {
+      Closeables.close(w, false)
+    }
+    /*
+        val indexedLikes = IndexedDatasetStore.readTuples(sc, "tmp/cf-data.txt", 2, ",", 0, 2, 1, "purchase")
+
+        val indexedViews = IndexedDatasetStore.readTuples(sc, "tmp/cf-data.txt", 2, ",", 0, 2, 1, "view")
+
+        val drmLikes = indexedLikes.matrix
+        val drmViews = indexedViews.matrix
+
+        // Now we could run cooccurrence analysis using the DRMs, instead we'll just fetch and print the matrices
+        val drmXCooccurrences = cooccurrences(drmLikes, randomSeed = 0xdeadbeef,
+          maxInterestingItemsPerThing = 100, maxNumInteractions = 500, Array(drmViews))
+
+        val inCoreViews = drmViews.collect
+        val inCoreLikes = drmLikes.collect
+        val inCoreIndicator = drmXCooccurrences(0).collect
+        val inCoreXIndicator = drmXCooccurrences(1).collect
+        println("\nLIKES:")
+        println(inCoreLikes)
+        println("\nVIEWS:")
+        println(inCoreViews)
+        println("\nINDICATOR MATRIX")
+        println(inCoreIndicator)
+        println("\nCROSS INDICATOR MATRIX")
+        println(inCoreXIndicator)
+    */
+    sc.stop()
+/*
+    //Clustered Spark and HDFS
+    ItemSimilarityDriver.main(Array(
+      "--input", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/cf-data.txt",
+      "--output", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/indicatorMatrices/",
+      "--master", "spark://occam4:7077",
+      "--filter1", "purchase",
+      "--filter2", "view",
+      "--inDelim", ",",
+      "--itemIDPosition", "2",
+      "--rowIDPosition", "0",
+      "--filterPosition", "1"
+    ))
+*/
+    //local multi-threaded Spark with HDFS
+/*    ItemSimilarityDriver.main(Array(
+    "--input", "hdfs://occam4:54310/user/pat/xrsj/ratings_data.txt",
+    "--output", "hdfs://occam4:54310/user/pat/xrsj/indicatorMatrices/",
+    "--master", "local[4]",
+    "--filter1", "purchase",
+    "--filter2", "view",
+    "--inDelim", ",",
+    "--itemIDPosition", "2",
+    "--rowIDPosition", "0",
+    "--filterPosition", "1"
+  ))
+*/
+
+    //local multi-threaded Spark with local FS
+    ItemSimilarityDriver.main(Array(
+      "--input", "tmp/cf-data.txt",
+      "--output", "tmp/indicatorMatrices/",
+      "--master", "local[4]",
+      "--filter1", "purchase",
+      "--filter2", "view",
+      "--inDelim", ",",
+      "--itemIDPosition", "2",
+      "--rowIDPosition", "0",
+      "--filterPosition", "1"
+    ))
+
+
+  }
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala b/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
new file mode 100644
index 0000000..c72a3b3
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/ItemSimilarityDriver.scala
@@ -0,0 +1,208 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package org.apache.mahout.drivers
+
+import org.apache.mahout.cf.CooccurrenceAnalysis
+
+/**
+  * Command line interface for [[org.apache.mahout.cf.CooccurrenceAnalysis.cooccurrences()]].
+  * Command line interface for [[org.apache.mahout.cf.CooccurrenceAnalysis.cooccurrences()]]. It text lines
+  * that contain (row id, column id, ...). The IDs are user specified strings which will be preserved in the
+  * output. The individual tuples will be accumulated into a matrix and [[org.apache.mahout.cf.CooccurrenceAnalysis.cooccurrences()]]
+  * will be used to calculate row-wise self-similarity, or when using filters, will generate two
+  * matrices and calculate both the self similarity of the primary matrix and the row-wise similarity of the primary
+  * to the secondary. Returns one or two directories of text files formatted as specified in the options.
+  * @todo Should also take two input streams and do cross similarity with not filter required.
+  */
+object ItemSimilarityDriver extends MahoutDriver {
+
+  private var options: Options = _
+  private var readStore1: TextDelimitedIndexedDatasetReader = _
+  private var readStore2: TextDelimitedIndexedDatasetReader = _
+  private var writeStore: TextDelimitedIndexedDatasetWriter = _
+
+/**
+  * @param args  Command line args, if empty a help message is printed.
+  */
+  override def main(args: Array[String]): Unit = {
+    val parser = new MahoutOptionParser[Options]("ItemSimilarity") {
+      head("ItemSimilarity", "Spark")
+      note("Input, output options")
+      opt[String]('i', "input") required() action { (x, options) =>
+        options.copy(input = x)
+      } text ("Path for input. It may be a filename or directory name and can be a local file path or an HDFS URI (required).")
+      opt[String]('o', "output") required() action { (x, options) =>
+        options.copy(output = x)
+      } text ("Output will be in sub-directories stored here so this must be a directory path (required).")
+      note("\nAlgorithm control options:")
+      opt[String]("master") abbr ("ma") text ("URL for the Spark Master. (optional). Default: 'local'") action { (x, options) =>
+        options.copy(master = x)
+      }
+      opt[Int]("maxPrefs") abbr ("mppu") action { (x, options) =>
+        options.copy(maxPrefs = x)
+      } text ("Max number of preferences to consider per user or item, users or items with more preferences will be sampled down (optional). Default: 500") validate { x =>
+        if (x > 0) success else failure("Option --maxPrefs must be > 0")
+      }
+      opt[Int]("minPrefs") abbr ("mp") action { (x, options) =>
+        options.copy(minPrefs = x)
+      } text ("Ignore users with less preferences than this (optional). Default: 1") validate { x =>
+        if (x > 0) success else failure("Option --minPrefs must be > 0")
+      }
+
+      opt[Int]('m', "maxSimilaritiesPerItem") action { (x, options) =>
+        options.copy(maxSimilaritiesPerItem = x)
+      } text ("Try to cap the number of similar items for each item to this number (optional). Default: 100") validate { x =>
+        if (x > 0) success else failure("Option --maxSimilaritiesPerItem must be > 0")
+      }
+      opt[Int]("randomSeed") abbr ("rs") action { (x, options) =>
+        options.copy(randomSeed = x)
+      } text ("Int to seed random number generator (optional). Default: Uses time to generate a seed") validate { x =>
+        if (x > 0) success else failure("Option --randomSeed must be > 0")
+      }
+      note("\nInput text file schema options:")
+      opt[String]("inDelim") abbr ("d") text ("Input delimiter character (optional). Default: '\\t'") action { (x, options) =>
+        options.copy(inDelim = x)
+      }
+      opt[String]("filter1") abbr ("f1") action { (x, options) =>
+        options.copy(filter1 = x)
+      } text ("String whose presence indicates a datum for the primary item set, can be a regex (optional). Default: no filtered is applied, all is used")
+      opt[String]("filter2") abbr ("f2") action { (x, options) =>
+        options.copy(filter2 = x)
+      } text ("String whose presence indicates a datum for the secondary item set, can be a regex (optional). Used in cross-cooccurrence. Default: no secondary filter is applied")
+      opt[Int]("rowIDPosition") abbr ("rc") action { (x, options) =>
+        options.copy(rowIDPosition = x)
+      } text ("Column number (0 based Int) containing the row ID string (optional). Default: 0") validate { x =>
+        if (x >= 0) success else failure("Option --rowIDColNum must be >= 0")
+      }
+      opt[Int]("itemIDPosition") abbr ("ic") action { (x, options) =>
+        options.copy(itemIDPosition = x)
+      } text ("Column number (0 based Int) containing the item ID string (optional). Default: 1") validate { x =>
+        if (x >= 0) success else failure("Option --itemIDColNum must be >= 0")
+      }
+      opt[Int]("filterPosition") abbr ("fc") action { (x, options) =>
+        options.copy(filterPosition = x)
+      } text ("Column number (0 based Int) containing the filter string (optional). Default: -1 for no filter") validate { x =>
+        if (x >= -1) success else failure("Option --filterColNum must be >= -1")
+      }
+      note("\nDefault input schema will accept: 'userID<tab>itemId' or 'userID<tab>itemID<tab>any-text...' and all rows will be used")
+      note("\nFile input options:")
+      opt[Unit]('r', "recursive") action { (_, options) =>
+        options.copy(recursive = true)
+      } text ("The input path should be searched recursively for files that match the filename pattern from -fp (optional), Default: false")
+      opt[String]("filenamePattern") abbr ("fp") action { (x, options) =>
+        options.copy(filenamePattern = x)
+      } text ("Regex to match in determining input files (optional). Default: filename in the --input option or '^part-.*' if --input is a directory")
+      note("\nOutput text file schema options:")
+      opt[String]("outDelim1") abbr ("od1") action { (x, options) =>
+        options.copy(outDelim1 = x)
+      } text ("Primary output inDelim value, used to separate row IDs from the similar items list (optional). Default: '\\t'")
+      opt[String]("outDelim2") abbr ("od2") action { (x, options) =>
+        options.copy(outDelim2 = x)
+      } text ("Secondary output inDelim value, used to separate item IDs from their values in the similar items list (optional). Default: ':'")
+      opt[String]("outDelim3") abbr ("od3") action { (x, options) =>
+        options.copy(outDelim3 = x)
+      } text ("Last inDelim value, used to separate (itemID:value) tuples in the similar items list. (optional). Default: ','")
+      note("\nDefault delimiters will produce output of the form: 'itemID1<tab>>itemID2:value2,itemID10:value10...'")
+      note("\nNote: Only the Log Likelihood Ratio (LLR) is supported as a similarity measure.\n")
+      help("help") abbr ("h") text ("prints this usage text\n")
+      checkConfig { c =>
+        if (c.filterPosition == c.itemIDPosition || c.filterPosition == c.rowIDPosition || c.rowIDPosition == c.itemIDPosition) failure("The row, item, and filter positions must be unique.") else success
+      }
+      checkConfig { c =>
+        if (c.filter1 != null && c.filter2 != null && c.filter1 == c.filter2) failure("If using filters they must be unique.") else success
+      }
+    }
+    parser.parse(args, Options()) map { opts =>
+      options = opts
+      process
+    }
+  }
+
+  private def readIndexedDatasets: Array[IndexedDataset] = {
+    val inFiles = FileSysUtils(options.input, options.filenamePattern, options.recursive ).uris
+    if(inFiles.isEmpty){Array()}else{
+      val indexedDataset1 = IndexedDataset(readStore1.readFrom(inFiles))
+      if (options.filterPosition != -1 && options.filter2 != null) {
+        val indexedDataset2 = IndexedDataset(readStore2.readFrom(inFiles))
+        Array(indexedDataset1, indexedDataset2)
+      } else {
+        Array(indexedDataset1)
+      }
+    }
+  }
+
+  override def start(masterUrl: String = options.master, appName: String = options.appName,
+                      customJars:Traversable[String] = Traversable.empty[String]): Unit = {
+    //todo: create and modify a SparkContext here, which can be passed in to mahoutSparkContext in the super.start
+    System.setProperty("spark.kryo.referenceTracking", "false")
+    System.setProperty("spark.kryoserializer.buffer.mb", "100")
+    System.setProperty("spark.executor.memory", "2g")
+    super.start(masterUrl, appName, customJars)
+    val readSchema1 = new Schema("delim" -> options.inDelim, "filter" -> options.filter1,  "rowIDPosition" -> options.rowIDPosition,  "columnIDPosition" -> options.itemIDPosition, "filterPosition" -> options.filterPosition)
+    if(options.filterPosition != -1 && options.filter2 != null) {
+      val readSchema2 = new Schema("delim" -> options.inDelim, "filter" -> options.filter2, "rowIDPosition" -> options.rowIDPosition, "columnIDPosition" -> options.itemIDPosition, "filterPosition" -> options.filterPosition)
+      readStore2 = new TextDelimitedIndexedDatasetReader(readSchema2, mc)
+    }
+    val writeSchema = new Schema("delim1" -> options.outDelim1, "delim2" -> options.outDelim2, "delim3" -> options.outDelim3)
+    readStore1 = new TextDelimitedIndexedDatasetReader(readSchema1, mc)
+    writeStore = new TextDelimitedIndexedDatasetWriter(writeSchema, mc)
+  }
+
+  override def process: Unit = {
+    start()
+
+    val indexedDatasets = readIndexedDatasets
+
+    val indicatorMatrices = CooccurrenceAnalysis.cooccurrences(indexedDatasets(0).matrix, options.randomSeed, options.maxSimilaritiesPerItem, options.maxPrefs, Array(indexedDatasets(1).matrix))
+
+    val selfIndicatorDataset = new IndexedDataset(indicatorMatrices(0), indexedDatasets(0).columnIDs, indexedDatasets(0).columnIDs) // self similarity
+
+    writeStore.writeTo(selfIndicatorDataset, options.output+"indicator-matrix")
+    if(indexedDatasets.length > 1){
+      val crossIndicatorDataset = new IndexedDataset(indicatorMatrices(1), indexedDatasets(0).columnIDs, indexedDatasets(1).columnIDs) // cross similarity
+      writeStore.writeTo(crossIndicatorDataset, options.output+"cross-indicator-matrix")
+    }
+
+    stop
+  }
+
+  //Default values go here, any '_' or null should be 'required' in the Parser or flags an unused option
+  // todo: support two input streams for cross-similarity, maybe assume one schema for both inputs
+  case class Options(
+    master: String = "local",
+    appName: String = "ItemSimilarityJob",
+    randomSeed: Int = System.currentTimeMillis().toInt,
+    recursive: Boolean = false,
+    input: String = null,
+    output: String = null,
+    filenamePattern: String = "^part-.*",
+    maxSimilaritiesPerItem: Int = 100,
+    maxPrefs: Int = 500,
+    minPrefs: Int = 1,
+    rowIDPosition: Int = 0,
+    itemIDPosition: Int = 1,
+    filterPosition: Int = -1,
+    filter1: String = null,
+    filter2: String = null,
+    inDelim: String = ",",
+    outDelim1: String = "\t",
+    outDelim2: String = ":",
+    outDelim3: String = ","
+  )
+
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala b/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
new file mode 100644
index 0000000..b8a79f8
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/MahoutDriver.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.drivers
+
+import org.apache.spark.SparkContext
+import org.apache.mahout.sparkbindings._
+
+/** Extend this class to create a Mahout CLI driver. Minimally you must override process and main. Also define a command line parser and default options or fill in the following template:
+  * {{{
+  *   object SomeDriver extends MahoutDriver {
+  *     override def main(args: Array[String]): Unit = {
+  *       val parser = new MahoutOptionParser[Options]("Job Name") {
+  *         head("Job Name", "Spark")
+  *         note("Various CLI options")
+  *         //see https://github.com/scopt/scopt for a good Scala option parser, which MahoutOptionParser extends
+  *       }
+  *       parser.parse(args, Options()) map { opts =>
+  *         options = opts
+  *         process
+  *       }
+  *     }
+  *
+  *     override def process: Unit = {
+  *       start()
+  *       //don't just stand there do something
+  *       stop
+  *     }
+  *
+  *     //Default values go here, any '_' or null should be 'required' in the Parser or flags an unused option
+  *     case class Options(
+  *       appName: String = "Job Name", ...
+  *     )
+  *   }
+  * }}}
+  */
+abstract class MahoutDriver {
+  protected var mc: SparkContext = _
+  /** Creates a Spark context to run the job inside.
+    * Creates a Spark context to run the job inside. Override to set the SparkConf values specific to the job, these must be set before the context is created.
+    * @param masterUrl Spark master URL
+    * @param appName  Name to display in Spark UI
+    * @param customJars List of paths to custom jars
+    * */
+  protected def start(masterUrl: String, appName: String,
+            customJars:Traversable[String] = Traversable.empty[String]) : Unit = {
+    mc = mahoutSparkContext(masterUrl, appName, customJars)
+  }
+
+  /** Override (optionally) for special cleanup */
+  protected def stop: Unit = {
+    mc.stop
+  }
+
+  /** This is wher you do the work, call start first, then before exiting call stop */
+  protected def process: Unit
+
+  /** Parse command line and call process*/
+  def main(args: Array[String]): Unit
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/MahoutOptionParser.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/MahoutOptionParser.scala b/spark/src/main/scala/org/apache/mahout/drivers/MahoutOptionParser.scala
new file mode 100644
index 0000000..8a337f5
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/MahoutOptionParser.scala
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.mahout.drivers
+
+import scopt.OptionParser
+
+/** Modifies default [[scopt.OptionParser]] to output long help-like usage + error message */
+class MahoutOptionParser[C](programName: String) extends OptionParser[C](programName: String) {
+  override def showUsageOnError = true
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/ReaderWriter.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/ReaderWriter.scala b/spark/src/main/scala/org/apache/mahout/drivers/ReaderWriter.scala
new file mode 100644
index 0000000..1179eef
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/ReaderWriter.scala
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.drivers
+
+import org.apache.mahout.sparkbindings.drm._
+import org.apache.spark.SparkContext._
+import org.apache.mahout.math.RandomAccessSparseVector
+import org.apache.spark.SparkContext
+import com.google.common.collect.{BiMap, HashBiMap}
+import scala.collection.JavaConversions._
+import org.apache.mahout.sparkbindings.DrmRdd
+import org.apache.mahout.math.drm.DrmLike
+
+
+/** Reader trait is abstract in the sense that the reader function must be defined by an extending trait, which also defines the type to be read.
+  * @tparam T type of object read, usually supplied by an extending trait.
+  * @todo the reader need not create both dictionaries but does at present. There are cases where one or the other dictionary is never used so saving the memory for a very large dictionary may be worth the optimization to specify which dictionaries are created.
+  */
+trait Reader[T]{
+  val sc: SparkContext
+  val readSchema: Schema
+  protected def reader(sc: SparkContext, readSchema: Schema, source: String): T
+  def readFrom(source: String): T = reader(sc, readSchema, source)
+}
+
+/** Writer trait is abstract in the sense that the writer method must be supplied by an extending trait, which also defines the type to be written.
+  * @tparam T
+  */
+trait Writer[T]{
+  val sc: SparkContext
+  val writeSchema: Schema
+  protected def writer(sc: SparkContext, writeSchema: Schema, dest: String, collection: T): Unit
+  def writeTo(collection: T, dest: String) = writer(sc, writeSchema, dest, collection)
+}
+
+/** Extends Reader trait to supply the [[org.apache.mahout.drivers.IndexedDataset]] as the type read and a reader function for reading text delimited files as described in the [[org.apache.mahout.drivers.Schema]]
+  */
+trait TDIndexedDatasetReader extends Reader[IndexedDataset]{
+  /** Read in text delimited tuples from all URIs in this comma delimited source String. 
+    * 
+    * @param sc context for the Spark job
+    * @param readSchema describes the delimiters and positions of values in the text delimited file.
+    * @param source comma delimited URIs of text files to be read into the [[org.apache.mahout.drivers.IndexedDataset]]
+    * @return
+    */
+  protected def reader(sc: SparkContext, readSchema: Schema, source: String): IndexedDataset = {
+    try {
+      val delimiter = readSchema("delim").asInstanceOf[String]
+      val rowIDPosition = readSchema("rowIDPosition").asInstanceOf[Int]
+      val columnIDPosition = readSchema("columnIDPosition").asInstanceOf[Int]
+      val filterPosition = readSchema("filterPosition").asInstanceOf[Int]
+      val filterBy = readSchema("filter").asInstanceOf[String]
+      //instance vars must be put into locally scoped vals when used in closures that are
+      //executed but Spark
+
+      assert(!source.isEmpty, {
+        println(this.getClass.toString + ": has no files to read")
+        throw new IllegalArgumentException
+      })
+
+      var columns = sc.textFile(source).map({ line => line.split(delimiter)})
+
+      columns = columns.filter({ tokens => tokens(filterPosition) == filterBy})
+
+      val interactions = columns.map({ tokens => tokens(rowIDPosition) -> tokens(columnIDPosition)})
+
+      interactions.cache()
+
+      val rowIDs = interactions.map({ case (rowID, _) => rowID}).distinct().collect()
+      val columnIDs = interactions.map({ case (_, columnID) => columnID}).distinct().collect()
+
+      val numRows = rowIDs.size
+      val numColumns = columnIDs.size
+
+      val rowIDDictionary = asOrderedDictionary(rowIDs)
+      val columnIDDictionary = asOrderedDictionary(columnIDs)
+
+      val rowIDDictionary_bcast = sc.broadcast(rowIDDictionary)
+      val columnIDDictionary_bcast = sc.broadcast(columnIDDictionary)
+
+      val indexedInteractions =
+        interactions.map({ case (rowID, columnID) =>
+          val rowIndex = rowIDDictionary_bcast.value.get(rowID).get
+          val columnIndex = columnIDDictionary_bcast.value.get(columnID).get
+
+          rowIndex -> columnIndex
+        }).groupByKey().map({ case (rowIndex, columnIndexes) =>
+          val row = new RandomAccessSparseVector(numColumns)
+          for (columnIndex <- columnIndexes) {
+            row.setQuick(columnIndex, 1.0)
+          }
+          rowIndex -> row
+        }).asInstanceOf[DrmRdd[Int]]
+
+      //todo: old API, val drmInteractions = new CheckpointedDrmBase[Int](indexedInteractions, numRows, numColumns)
+      val drmInteractions = new CheckpointedDrmSpark[Int](indexedInteractions, numRows, numColumns)
+
+      IndexedDataset(drmInteractions, rowIDDictionary, columnIDDictionary)
+
+    } catch {
+      case cce: ClassCastException => {
+        println(this.getClass.toString + ": Schema has illegal values"); throw cce
+      }
+    }
+  }
+
+  private def asOrderedDictionary(entries: Array[String]): BiMap[String, Int] = {
+    var dictionary: BiMap[String, Int] = HashBiMap.create()
+    var index = 0
+    for (entry <- entries) {
+      dictionary.forcePut(entry, index)
+      index += 1
+    }
+    dictionary
+  }
+}
+
+trait TDIndexedDatasetWriter extends Writer[IndexedDataset]{
+  /** Read in text delimited tuples from all URIs in this comma delimited source String. 
+    *
+    * @param sc context for the Spark job
+    * @param writeSchema describes the delimiters and positions of values in the output text delimited file.
+    * @param dest directory to write text delimited version of [[org.apache.mahout.drivers.IndexedDataset]]
+    */
+  protected def writer(sc: SparkContext, writeSchema: Schema, dest: String, indexedDataset: IndexedDataset): Unit = {
+    try {
+      val outDelim1 = writeSchema("delim1").asInstanceOf[String]
+      val outDelim2 = writeSchema("delim2").asInstanceOf[String]
+      val outDelim3 = writeSchema("delim3").asInstanceOf[String]
+      //instance vars must be put into locally scoped vals when put into closures that are
+      //executed but Spark
+      assert (indexedDataset != null, {println(this.getClass.toString+": has no indexedDataset to write"); throw new IllegalArgumentException })
+      assert (!dest.isEmpty, {println(this.getClass.toString+": has no destination or indextedDataset to write"); throw new IllegalArgumentException})
+      val matrix: DrmLike[Int] = indexedDataset.matrix
+      val rowIDDictionary: BiMap[String, Int] = indexedDataset.rowIDs
+      val columnIDDictionary: BiMap[String, Int] = indexedDataset.columnIDs
+      matrix.rdd.map({ case (rowID, itemVector) =>
+        var line: String = rowIDDictionary.inverse.get(rowID) + outDelim1
+        for (item <- itemVector.nonZeroes()) {
+          line += columnIDDictionary.inverse.get(item.index) + outDelim2 + item.get + outDelim3
+        }
+        line.dropRight(1)
+      })
+        .saveAsTextFile(dest)
+    }catch{
+      case cce: ClassCastException => {println(this.getClass.toString+": Schema has illegal values"); throw cce}
+    }
+  }
+}
+
+/** A combined trait that reads and writes */
+trait TDIndexedDatasetReaderWriter extends TDIndexedDatasetReader with TDIndexedDatasetWriter
+
+/** Reads text delimited files into an IndexedDataset. Classes are needed to supply trait params in their constructor.
+  * @param readSchema describes the delimiters and position of values in the text delimited file to be read.
+  * @param sc Spark context for reading files
+  * @note The source files are supplied to the readFrom trait method.
+  * */
+class TextDelimitedIndexedDatasetReader(val readSchema: Schema, val sc: SparkContext) extends TDIndexedDatasetReader
+
+/** Writes  text delimited files into an IndexedDataset. Classes are needed to supply trait params in their constructor.
+  * @param writeSchema describes the delimiters and position of values in the text delimited file(s) written.
+  * @param sc Spark context for reading files
+  * @note the destination if supplied to the writeTo trait method
+  * */
+class TextDelimitedIndexedDatasetWriter(val writeSchema: Schema, val sc: SparkContext) extends TDIndexedDatasetWriter
+
+/** Reads and writes text delimited files to/from an IndexedDataset. Classes are needed to supply trait params in their constructor.
+  * @param readSchema describes the delimiters and position of values in the text delimited file(s) to be read.
+  * @param writeSchema describes the delimiters and position of values in the text delimited file(s) written.
+  * @param sc Spark context for reading the files
+  * */
+class TextDelimitedIndexedDatasetReaderWriter(val readSchema: Schema, val writeSchema: Schema, val sc: SparkContext) extends TDIndexedDatasetReaderWriter

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/main/scala/org/apache/mahout/drivers/Schema.scala
----------------------------------------------------------------------
diff --git a/spark/src/main/scala/org/apache/mahout/drivers/Schema.scala b/spark/src/main/scala/org/apache/mahout/drivers/Schema.scala
new file mode 100644
index 0000000..13fa292
--- /dev/null
+++ b/spark/src/main/scala/org/apache/mahout/drivers/Schema.scala
@@ -0,0 +1,30 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.drivers
+
+import scala.collection.mutable.HashMap
+
+/** Syntactic sugar for HashMap[String, Any]
+  *
+  * @param params list of mappings for instantiation {{{val mySchema = new Schema("one" -> 1, "two" -> "2", ...)}}}
+  */
+class Schema(params: Tuple2[String, Any]*) extends HashMap[String, Any] {
+  //todo: this require a mutable HashMap, do we care?
+  this ++= params
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/test/scala/org/apache/mahout/drivers/IndexedDatasetTest.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/drivers/IndexedDatasetTest.scala b/spark/src/test/scala/org/apache/mahout/drivers/IndexedDatasetTest.scala
new file mode 100644
index 0000000..fb13361
--- /dev/null
+++ b/spark/src/test/scala/org/apache/mahout/drivers/IndexedDatasetTest.scala
@@ -0,0 +1,25 @@
+package org.apache.mahout.drivers
+
+import org.scalatest.FunSuite
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class IndexedDatasetTest extends FunSuite {
+  //todo: put some tests here!
+
+}

http://git-wip-us.apache.org/repos/asf/mahout/blob/74b9921c/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriver$Test.scala
----------------------------------------------------------------------
diff --git a/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriver$Test.scala b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriver$Test.scala
new file mode 100644
index 0000000..c1ae4ab
--- /dev/null
+++ b/spark/src/test/scala/org/apache/mahout/drivers/ItemSimilarityDriver$Test.scala
@@ -0,0 +1,180 @@
+package org.apache.mahout.drivers
+
+import org.scalatest.{BeforeAndAfter, FunSuite}
+
+import org.apache.mahout.sparkbindings._
+import java.io.{FileWriter, BufferedWriter}
+import com.google.common.io.Closeables
+import org.apache.spark.SparkContext
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+class ItemSimilarityDriver$Test extends FunSuite {
+  private var sc: SparkContext = _
+
+
+  test ("Running some text delimited files through the import/cooccurrence/export"){
+
+    val exampleCsvlogStatements = Array(
+      "12569537329,user1,item1,\"view\"",
+      "12569537329,user1,item2,\"view\"",
+      "12569537329,user1,item2,\"like\"",
+      "12569537329,user1,item3,\"like\"",
+      "12569537329,user2,item2,\"view\"",
+      "12569537329,user2,item2,\"like\"",
+      "12569537329,user3,item1,\"like\"",
+      "12569537329,user3,item3,\"view\"",
+      "12569537329,user3,item3,\"like\""
+    )
+
+    val exampleTsvLogStatements = Array(
+      "12569537329\tuser1\titem1\t\"view\"",
+      "12569537329\tuser1\titem2\t\"view\"",
+      "12569537329\tuser1\titem2\t\"like\"",
+      "12569537329\tuser1\titem3\t\"like\"",
+      "12569537329\tuser2\titem2\t\"view\"",
+      "12569537329\tuser2\titem2\t\"like\"",
+      "12569537329\tuser3\titem1\t\"like\"",
+      "12569537329\tuser3\titem3\t\"view\"",
+      "12569537329\tuser3\titem3\t\"like\""
+    )
+
+
+    val csvLogStatements = Array(
+      "u1,purchase,iphone",
+      "u1,purchase,ipad",
+      "u2,purchase,nexus",
+      "u2,purchase,galaxy",
+      "u3,purchase,surface",
+      "u4,purchase,iphone",
+      "u4,purchase,galaxy",
+      "u1,view,iphone",
+      "u1,view,ipad",
+      "u1,view,nexus",
+      "u1,view,galaxy",
+      "u2,view,iphone",
+      "u2,view,ipad",
+      "u2,view,nexus",
+      "u2,view,galaxy",
+      "u3,view,surface",
+      "u3,view,nexus",
+      "u4,view,iphone",
+      "u4,view,ipad",
+      "u4,view,galaxy"
+    )
+
+    val tsvLogStatements = Array(
+      "u1\tpurchase\tiphone",
+      "u1\tpurchase\tipad",
+      "u2\tpurchase\tnexus",
+      "u2\tpurchase\tgalaxy",
+      "u3\tpurchase\tsurface",
+      "u4\tpurchase\tiphone",
+      "u4\tpurchase\tgalaxy",
+      "u1\tview\tiphone",
+      "u1\tview\tipad",
+      "u1\tview\tnexus",
+      "u1\tview\tgalaxy",
+      "u2\tview\tiphone",
+      "u2\tview\tipad",
+      "u2\tview\tnexus",
+      "u2\tview\tgalaxy",
+      "u3\tview\tsurface",
+      "u3\tview\tnexus",
+      "u4\tview\tiphone",
+      "u4\tview\tipad",
+      "u4\tview\tgalaxy"
+    )
+
+    var w: BufferedWriter = null
+    //try {
+      w = new BufferedWriter(new FileWriter("/tmp/cf-data.txt"))
+      w.write(csvLogStatements.mkString("\n"))
+    //} finally {
+      Closeables.close(w, false)
+    //}
+    /*
+        val indexedLikes = IndexedDatasetStore.readTuples(sc, "tmp/cf-data.txt", 2, ",", 0, 2, 1, "purchase")
+
+        val indexedViews = IndexedDatasetStore.readTuples(sc, "/tmp/cf-data.txt", 2, ",", 0, 2, 1, "view")
+
+        val drmLikes = indexedLikes.matrix
+        val drmViews = indexedViews.matrix
+
+        // Now we could run cooccurrence analysis using the DRMs, instead we'll just fetch and print the matrices
+        val drmXCooccurrences = cooccurrences(drmLikes, randomSeed = 0xdeadbeef,
+          maxInterestingItemsPerThing = 100, maxNumInteractions = 500, Array(drmViews))
+
+        val inCoreViews = drmViews.collect
+        val inCoreLikes = drmLikes.collect
+        val inCoreIndicator = drmXCooccurrences(0).collect
+        val inCoreXIndicator = drmXCooccurrences(1).collect
+        println("\nLIKES:")
+        println(inCoreLikes)
+        println("\nVIEWS:")
+        println(inCoreViews)
+        println("\nINDICATOR MATRIX")
+        println(inCoreIndicator)
+        println("\nCROSS INDICATOR MATRIX")
+        println(inCoreXIndicator)
+    */
+
+    /*
+        //Clustered Spark and HDFS
+        ItemSimilarityDriver.main(Array(
+          "--input", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/cf-data.txt",
+          "--output", "hdfs://occam4:54310/user/pat/spark-itemsimilarity/indicatorMatrices/",
+          "--master", "spark://occam4:7077",
+          "--filter1", "purchase",
+          "--filter2", "view",
+          "--inDelim", ",",
+          "--itemIDPosition", "2",
+          "--rowIDPosition", "0",
+          "--filterPosition", "1"
+        ))
+    */
+    //local multi-threaded Spark with HDFS using large dataset
+    /*    ItemSimilarityDriver.main(Array(
+        "--input", "hdfs://occam4:54310/user/pat/xrsj/ratings_data.txt",
+        "--output", "hdfs://occam4:54310/user/pat/xrsj/indicatorMatrices/",
+        "--master", "local[4]",
+        "--filter1", "purchase",
+        "--filter2", "view",
+        "--inDelim", ",",
+        "--itemIDPosition", "2",
+        "--rowIDPosition", "0",
+        "--filterPosition", "1"
+      ))
+    */
+
+    //local multi-threaded Spark with local FS
+    ItemSimilarityDriver.main(Array(
+      "--input", "/tmp/cf-data.txt",
+      "--output", "tmp/indicatorMatrices/",
+      "--master", "local[4]",
+      "--filter1", "purchase",
+      "--filter2", "view",
+      "--inDelim", ",",
+      "--itemIDPosition", "2",
+      "--rowIDPosition", "0",
+      "--filterPosition", "1"
+    ))
+
+  }
+
+}