You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sm...@apache.org on 2013/06/01 20:20:44 UTC

svn commit: r1488552 - in /mahout/trunk: CHANGELOG core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java

Author: smarthi
Date: Sat Jun  1 18:20:43 2013
New Revision: 1488552

URL: http://svn.apache.org/r1488552
Log:
MAHOUT-1047: CVB hangs after completion

Modified:
    mahout/trunk/CHANGELOG
    mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java
    mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java

Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1488552&r1=1488551&r2=1488552&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Sat Jun  1 18:20:43 2013
@@ -2,6 +2,8 @@ Mahout Change Log
 
 Release 0.8 - unreleased
 
+  MAHOUT-1047: CVB hangs after completion (Angel Martinez Gonzalez via smarthi)
+  
   MAHOUT-1235: ParallelALSFactorizationJob does not use VectorSumCombiner (ssc)
 
   MAHOUT-1230: SparceMatrix.clone() is not deep copy (Maysam Yabandeh via tdunning)

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java?rev=1488552&r1=1488551&r2=1488552&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java Sat Jun  1 18:20:43 2013
@@ -102,6 +102,7 @@ public class ModelTrainer {
         workQueue);
     threadPool.allowCoreThreadTimeOut(false);
     threadPool.prestartAllCoreThreads();
+    writeModel.reset();
   }
 
   public void train(VectorIterable matrix, VectorIterable docTopicCounts) {
@@ -240,13 +241,16 @@ public class ModelTrainer {
       long newTime = System.nanoTime();
       log.info("threadpool took: {}ms", (newTime - startTime) / 1.0e6);
       startTime = newTime;
-      writeModel.awaitTermination();
+      readModel.stop();
       newTime = System.nanoTime();
-      log.info("writeModel.awaitTermination() took {}ms", (newTime - startTime) / 1.0e6);
+      log.info("readModel.stop() took {}ms", (newTime - startTime) / 1.0e6);
+      startTime = newTime;
+      writeModel.stop();
+      newTime = System.nanoTime();
+      log.info("writeModel.stop() took {}ms", (newTime - startTime) / 1.0e6);
       TopicModel tmpModel = writeModel;
       writeModel = readModel;
       readModel = tmpModel;
-      writeModel.reset();
     } catch (InterruptedException e) {
       log.error("Interrupted shutting down!", e);
     }

Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java?rev=1488552&r1=1488551&r2=1488552&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java Sat Jun  1 18:20:43 2013
@@ -77,6 +77,7 @@ public class TopicModel implements Confi
 
   private final Sampler sampler;
   private final int numThreads;
+  private ThreadPoolExecutor threadPool;
   private Updater[] updaters;
 
   public int getNumTerms() {
@@ -153,7 +154,7 @@ public class TopicModel implements Confi
   }
 
   private void initializeThreadPool() {
-    ThreadPoolExecutor threadPool = new ThreadPoolExecutor(numThreads, numThreads, 0, TimeUnit.SECONDS,
+    threadPool = new ThreadPoolExecutor(numThreads, numThreads, 0, TimeUnit.SECONDS,
                                                            new ArrayBlockingQueue<Runnable>(numThreads * 10));
     threadPool.allowCoreThreadTimeOut(false);
     updaters = new Updater[numThreads];
@@ -246,13 +247,23 @@ public class TopicModel implements Confi
       topicTermCounts.assignRow(x, new SequentialAccessSparseVector(numTerms));
     }
     topicSums.assign(1.0);
-    initializeThreadPool();
+    if(threadPool.isTerminated()) {
+      initializeThreadPool();
+    }
   }
 
-  public void awaitTermination() {
+  public void stop() {
     for (Updater updater : updaters) {
       updater.shutdown();
     }
+    threadPool.shutdown();
+    try {
+      if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
+        log.warn("Threadpool timed out on await termination - jobs still running!");
+      }
+    } catch (InterruptedException e) {
+        log.error("Interrupted shutting down!", e);
+    }
   }
 
   public void renormalize() {
@@ -275,12 +286,12 @@ public class TopicModel implements Confi
         docTopicModelRow.setQuick(e.index(), docTopicModelRow.getQuick(e.index()) * e.get());
       }
     }
-    // now recalculate p(topic|doc) by summing contributions from all of pTopicGivenTerm
+    // now recalculate \(p(topic|doc)\) by summing contributions from all of pTopicGivenTerm
     topics.assign(0.0);
     for (int x = 0; x < numTopics; x++) {
       topics.set(x, docTopicModel.viewRow(x).norm(1));
     }
-    // now renormalize so that sum_x(p(x|doc)) = 1
+    // now renormalize so that \(sum_x(p(x|doc))\) = 1
     topics.assign(Functions.mult(1 / topics.norm(1)));
   }
 
@@ -326,14 +337,14 @@ public class TopicModel implements Confi
   }
 
   /**
-   * Computes {@code p(topic x | term a, document i)} distributions given input document {@code i}.
-   * {@code pTGT[x][a]} is the (un-normalized) {@code p(x|a,i)}, or if docTopics is {@code null},
-   * {@code p(a|x)} (also un-normalized).
+   * Computes {@code \(p(topic x | term a, document i)\)} distributions given input document {@code i}.
+   * {@code \(pTGT[x][a]\)} is the (un-normalized) {@code \(p(x|a,i)\)}, or if docTopics is {@code null},
+   * {@code \(p(a|x)\)} (also un-normalized).
    *
-   * @param document doc-term vector encoding {@code w(term a|document i)}.
+   * @param document doc-term vector encoding {@code \(w(term a|document i)\)}.
    * @param docTopics {@code docTopics[x]} is the overall weight of topic {@code x} in given
    *          document. If {@code null}, a topic weight of {@code 1.0} is used for all topics.
-   * @param termTopicDist storage for output {@code p(x|a,i)} distributions.
+   * @param termTopicDist storage for output {@code \(p(x|a,i)\)} distributions.
    */
   private void pTopicGivenTerm(Vector document, Vector docTopics, Matrix termTopicDist) {
     // for each topic x
@@ -360,7 +371,7 @@ public class TopicModel implements Confi
   }
 
   /**
-   * sum_x sum_a (c_ai * log(p(x|i) * p(a|x)))
+   * \(sum_x sum_a (c_ai * log(p(x|i) * p(a|x)))\)
    */
   public double perplexity(Vector document, Vector docTopics) {
     double perplexity = 0;