You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by nl...@apache.org on 2017/06/17 15:26:45 UTC

[5/9] incubator-spot git commit: Made changes to SpotLDAWrapper and ml_ops.sh based on feedback from @NathanSegerlind Fixed inline comments format, added one space after //.

Made changes to SpotLDAWrapper and ml_ops.sh based on feedback from @NathanSegerlind
Fixed inline comments format, added one space after //.


Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/9a71071b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/9a71071b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/9a71071b

Branch: refs/heads/master
Commit: 9a71071bf94194f0eabf8b65e7e1af9f9074f59a
Parents: f95e395
Author: Ricardo Barona <ri...@intel.com>
Authored: Tue May 30 16:59:10 2017 -0500
Committer: Ricardo Barona <ri...@intel.com>
Committed: Fri Jun 16 17:53:27 2017 -0500

----------------------------------------------------------------------
 spot-ml/ml_test.sh                              |  1 -
 .../org/apache/spot/lda/SpotLDAWrapper.scala    | 37 ++++++++++++--------
 2 files changed, 23 insertions(+), 15 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/9a71071b/spot-ml/ml_test.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_test.sh b/spot-ml/ml_test.sh
index e6bbe31..807b46c 100755
--- a/spot-ml/ml_test.sh
+++ b/spot-ml/ml_test.sh
@@ -21,7 +21,6 @@
 
 DSOURCE=$1
 RAWDATA_PATH=$2
-LDAOPTIMIZER="online"
 
 # read in variables (except for date) from etc/.conf file
 

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/9a71071b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
index f879ad7..7a5266c 100644
--- a/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/lda/SpotLDAWrapper.scala
@@ -75,7 +75,7 @@ object SpotLDAWrapper {
       .toDF(DocumentName, DocumentNumber)
       .cache
 
-    //Structure corpus so that the index is the docID, values are the vectors of word occurrences in that doc
+    // Structure corpus so that the index is the docID, values are the vectors of word occurrences in that doc
     val ldaCorpus: RDD[(Long, Vector)] =
       formatSparkLDAInput(docWordCountCache,
         documentDictionary,
@@ -86,7 +86,7 @@ object SpotLDAWrapper {
 
     docWordCountCache.unpersist()
 
-    //Instantiate optimizer based on input
+    // Instantiate optimizer based on input
     val ldaOptimizer = ldaOptimizerOption match {
       case "em" => new EMLDAOptimizer
       case "online" => new OnlineLDAOptimizer().setOptimizeDocConcentration(true).setMiniBatchFraction({
@@ -100,7 +100,7 @@ object SpotLDAWrapper {
     logger.info(s"Running Spark LDA with params alpha = $ldaAlpha beta = $ldaBeta " +
       s"Max iterations = $maxIterations Optimizer = $ldaOptimizerOption")
 
-    //Set LDA params from input args
+    // Set LDA params from input args
     val lda =
       new LDA()
         .setK(topicCount)
@@ -117,12 +117,16 @@ object SpotLDAWrapper {
 
     val (wordTopicMat, docTopicDist) = ldaOptimizer match {
       case _: EMLDAOptimizer => {
-         val ldaModel = lda.run(ldaCorpus).asInstanceOf[DistributedLDAModel]//.toLocal
+        val ldaModel = lda.run(ldaCorpus).asInstanceOf[DistributedLDAModel]
 
-        //Get word topic mix: columns = topic (in no guaranteed order), rows = words (# rows = vocab size)
+        // Get word topic mix, from Spark documentation:
+        // Inferred topics, where each topic is represented by a distribution over terms.
+        // This is a matrix of size vocabSize x k, where each column is a topic.
+        // No guarantees are given about the ordering of the topics.
         val wordTopicMat: Matrix = ldaModel.topicsMatrix
 
-        //Topic distribution: for each document, return distribution (vector) over topics for that docs
+        // Topic distribution: for each document, return distribution (vector) over topics for that docs where entry
+        // i is the fraction of the document which belongs to topic i
         val docTopicDist: RDD[(Long, Vector)] = ldaModel.topicDistributions
 
         (wordTopicMat, docTopicDist)
@@ -132,10 +136,14 @@ object SpotLDAWrapper {
       case _: OnlineLDAOptimizer => {
         val ldaModel = lda.run(ldaCorpus).asInstanceOf[LocalLDAModel]
 
-        //Get word topic mix: columns = topic (in no guaranteed order), rows = words (# rows = vocab size)
+        // Get word topic mix, from Spark documentation:
+        // Inferred topics, where each topic is represented by a distribution over terms.
+        // This is a matrix of size vocabSize x k, where each column is a topic.
+        // No guarantees are given about the ordering of the topics.
         val wordTopicMat: Matrix = ldaModel.topicsMatrix
 
-        //Topic distribution: for each document, return distribution (vector) over topics for that docs
+        // Topic distribution: for each document, return distribution (vector) over topics for that docs where entry
+        // i is the fraction of the document which belongs to topic i
         val docTopicDist: RDD[(Long, Vector)] = ldaModel.topicDistributions(ldaCorpus)
 
         (wordTopicMat, docTopicDist)
@@ -144,18 +152,19 @@ object SpotLDAWrapper {
 
     }
 
-    //Create doc results from vector: convert docID back to string, convert vector of probabilities to array
+    // Create doc results from vector: convert docID back to string, convert vector of probabilities to array
     val docToTopicMixDF =
       formatSparkLDADocTopicOutput(docTopicDist, documentDictionary, sqlContext, precisionUtility)
 
     documentDictionary.unpersist()
 
-    //Create word results from matrix: convert matrix to sequence, wordIDs back to strings, sequence of probabilities to array
+    // Create word results from matrix: convert matrix to sequence, wordIDs back to strings, sequence of
+    // probabilities to array
     val revWordMap: Map[Int, String] = wordDictionary.map(_.swap)
 
     val wordResults = formatSparkLDAWordOutput(wordTopicMat, revWordMap)
 
-    //Create output object
+    // Create output object
     SpotLDAOutput(docToTopicMixDF, wordResults)
   }
 
@@ -174,8 +183,8 @@ object SpotLDAWrapper {
       .map({ case SpotLDAInput(doc, word, count) => (doc, word, count) })
       .toDF(DocumentName, WordName, WordNameWordCount)
 
-    //Convert SpotSparkLDAInput into desired format for Spark LDA: (doc, word, count) -> word count per doc, where RDD
-    //is indexed by DocID
+    // Convert SpotSparkLDAInput into desired format for Spark LDA: (doc, word, count) -> word count per doc, where RDD
+    // is indexed by DocID
     val wordCountsPerDocDF = docWordCountDF
       .join(documentDictionary, docWordCountDF(DocumentName) === documentDictionary(DocumentName))
       .drop(documentDictionary(DocumentName))
@@ -188,7 +197,7 @@ object SpotLDAWrapper {
       .map({ case Row(documentId: Long, wordId: Int, wordCount: Int) => (documentId.toLong, (wordId, wordCount.toDouble)) })
       .groupByKey
 
-    //Sum of distinct words in each doc (words will be repeated between different docs), used for sparse vec size
+    // Sum of distinct words in each doc (words will be repeated between different docs), used for sparse vec size
     val numUniqueWords = wordDictionary.size
     val ldaInput: RDD[(Long, Vector)] = wordCountsPerDoc
       .mapValues({ case vs => Vectors.sparse(numUniqueWords, vs.toSeq) })