You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sm...@apache.org on 2013/06/01 20:20:44 UTC
svn commit: r1488552 - in /mahout/trunk: CHANGELOG
core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java
core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java
Author: smarthi
Date: Sat Jun 1 18:20:43 2013
New Revision: 1488552
URL: http://svn.apache.org/r1488552
Log:
MAHOUT-1047: CVB hangs after completion
Modified:
mahout/trunk/CHANGELOG
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java
Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1488552&r1=1488551&r2=1488552&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Sat Jun 1 18:20:43 2013
@@ -2,6 +2,8 @@ Mahout Change Log
Release 0.8 - unreleased
+ MAHOUT-1047: CVB hangs after completion (Angel Martinez Gonzalez via smarthi)
+
MAHOUT-1235: ParallelALSFactorizationJob does not use VectorSumCombiner (ssc)
MAHOUT-1230: SparceMatrix.clone() is not deep copy (Maysam Yabandeh via tdunning)
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java?rev=1488552&r1=1488551&r2=1488552&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/ModelTrainer.java Sat Jun 1 18:20:43 2013
@@ -102,6 +102,7 @@ public class ModelTrainer {
workQueue);
threadPool.allowCoreThreadTimeOut(false);
threadPool.prestartAllCoreThreads();
+ writeModel.reset();
}
public void train(VectorIterable matrix, VectorIterable docTopicCounts) {
@@ -240,13 +241,16 @@ public class ModelTrainer {
long newTime = System.nanoTime();
log.info("threadpool took: {}ms", (newTime - startTime) / 1.0e6);
startTime = newTime;
- writeModel.awaitTermination();
+ readModel.stop();
newTime = System.nanoTime();
- log.info("writeModel.awaitTermination() took {}ms", (newTime - startTime) / 1.0e6);
+ log.info("readModel.stop() took {}ms", (newTime - startTime) / 1.0e6);
+ startTime = newTime;
+ writeModel.stop();
+ newTime = System.nanoTime();
+ log.info("writeModel.stop() took {}ms", (newTime - startTime) / 1.0e6);
TopicModel tmpModel = writeModel;
writeModel = readModel;
readModel = tmpModel;
- writeModel.reset();
} catch (InterruptedException e) {
log.error("Interrupted shutting down!", e);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java?rev=1488552&r1=1488551&r2=1488552&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java Sat Jun 1 18:20:43 2013
@@ -77,6 +77,7 @@ public class TopicModel implements Confi
private final Sampler sampler;
private final int numThreads;
+ private ThreadPoolExecutor threadPool;
private Updater[] updaters;
public int getNumTerms() {
@@ -153,7 +154,7 @@ public class TopicModel implements Confi
}
private void initializeThreadPool() {
- ThreadPoolExecutor threadPool = new ThreadPoolExecutor(numThreads, numThreads, 0, TimeUnit.SECONDS,
+ threadPool = new ThreadPoolExecutor(numThreads, numThreads, 0, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(numThreads * 10));
threadPool.allowCoreThreadTimeOut(false);
updaters = new Updater[numThreads];
@@ -246,13 +247,23 @@ public class TopicModel implements Confi
topicTermCounts.assignRow(x, new SequentialAccessSparseVector(numTerms));
}
topicSums.assign(1.0);
- initializeThreadPool();
+ if(threadPool.isTerminated()) {
+ initializeThreadPool();
+ }
}
- public void awaitTermination() {
+ public void stop() {
for (Updater updater : updaters) {
updater.shutdown();
}
+ threadPool.shutdown();
+ try {
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
+ log.warn("Threadpool timed out on await termination - jobs still running!");
+ }
+ } catch (InterruptedException e) {
+ log.error("Interrupted shutting down!", e);
+ }
}
public void renormalize() {
@@ -275,12 +286,12 @@ public class TopicModel implements Confi
docTopicModelRow.setQuick(e.index(), docTopicModelRow.getQuick(e.index()) * e.get());
}
}
- // now recalculate p(topic|doc) by summing contributions from all of pTopicGivenTerm
+ // now recalculate \(p(topic|doc)\) by summing contributions from all of pTopicGivenTerm
topics.assign(0.0);
for (int x = 0; x < numTopics; x++) {
topics.set(x, docTopicModel.viewRow(x).norm(1));
}
- // now renormalize so that sum_x(p(x|doc)) = 1
+ // now renormalize so that \(sum_x(p(x|doc))\) = 1
topics.assign(Functions.mult(1 / topics.norm(1)));
}
@@ -326,14 +337,14 @@ public class TopicModel implements Confi
}
/**
- * Computes {@code p(topic x | term a, document i)} distributions given input document {@code i}.
- * {@code pTGT[x][a]} is the (un-normalized) {@code p(x|a,i)}, or if docTopics is {@code null},
- * {@code p(a|x)} (also un-normalized).
+ * Computes {@code \(p(topic x | term a, document i)\)} distributions given input document {@code i}.
+ * {@code \(pTGT[x][a]\)} is the (un-normalized) {@code \(p(x|a,i)\)}, or if docTopics is {@code null},
+ * {@code \(p(a|x)\)} (also un-normalized).
*
- * @param document doc-term vector encoding {@code w(term a|document i)}.
+ * @param document doc-term vector encoding {@code \(w(term a|document i)\)}.
* @param docTopics {@code docTopics[x]} is the overall weight of topic {@code x} in given
* document. If {@code null}, a topic weight of {@code 1.0} is used for all topics.
- * @param termTopicDist storage for output {@code p(x|a,i)} distributions.
+ * @param termTopicDist storage for output {@code \(p(x|a,i)\)} distributions.
*/
private void pTopicGivenTerm(Vector document, Vector docTopics, Matrix termTopicDist) {
// for each topic x
@@ -360,7 +371,7 @@ public class TopicModel implements Confi
}
/**
- * sum_x sum_a (c_ai * log(p(x|i) * p(a|x)))
+ * \(sum_x sum_a (c_ai * log(p(x|i) * p(a|x)))\)
*/
public double perplexity(Vector document, Vector docTopics) {
double perplexity = 0;