You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sm...@apache.org on 2013/12/01 18:42:04 UTC
svn commit: r1546825 - in /mahout/trunk: ./
core/src/main/java/org/apache/mahout/cf/taste/impl/eval/
core/src/main/java/org/apache/mahout/classifier/sgd/
core/src/main/java/org/apache/mahout/clustering/classify/
core/src/main/java/org/apache/mahout/clu...
Author: smarthi
Date: Sun Dec 1 17:42:03 2013
New Revision: 1546825
URL: http://svn.apache.org/r1546825
Log:
MAHOUT-1345: Enable randomised testing for all Mahout modules
Modified:
mahout/trunk/.gitignore
mahout/trunk/CHANGELOG
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java
mahout/trunk/core/src/main/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegression.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/classify/ClusterClassificationDriver.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0Mapper.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0PerplexityMapper.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java
mahout/trunk/core/src/main/java/org/apache/mahout/ep/EvolutionaryProcess.java
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ALSWRFactorizerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ParallelSGDFactorizerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegressionTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/ModelSerializerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/ep/EvolutionaryProcessTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java
mahout/trunk/examples/pom.xml
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneStorageConfiguration.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriver.java
mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java
mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/nlp/collocations/llr/BloomTokenFilterTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfoTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java
mahout/trunk/math/src/main/java/org/apache/mahout/math/decomposer/AsyncEigenVerifier.java
mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java
mahout/trunk/math/src/test/java/org/apache/mahout/math/decomposer/hebbian/TestHebbianSolver.java
Modified: mahout/trunk/.gitignore
URL: http://svn.apache.org/viewvc/mahout/trunk/.gitignore?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/.gitignore (original)
+++ mahout/trunk/.gitignore Sun Dec 1 17:42:03 2013
@@ -1,3 +1,7 @@
+output-asf-email-examples/
+.checkstyle
+.ruleset
+.pmd
.classpath
.project
.settings/
Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Sun Dec 1 17:42:03 2013
@@ -16,6 +16,8 @@ Release 0.9 - unreleased
MAHOUT-1347: Add Streaming K-Means clustering algorithm to examples/bin/cluster-reuters.sh (smarthi)
+ MAHOUT-1345: Enable randomised testing for all Mahout modules (Isabel, sslavic, Frank Scholten ,smarthi)
+
MAHOUT-1343: JSON output format support in cluster dumper (Telvis Calhoun via sslavic)
MAHOUT-1333: Fixed examples bin directory permissions in distribution archives (Mike Percy via sslavic)
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java Sun Dec 1 17:42:03 2013
@@ -26,6 +26,7 @@ import java.util.concurrent.ExecutionExc
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Lists;
@@ -198,12 +199,19 @@ public abstract class AbstractDifference
for (Future<Void> future : futures) {
future.get();
}
+
} catch (InterruptedException ie) {
throw new TasteException(ie);
} catch (ExecutionException ee) {
throw new TasteException(ee.getCause());
}
+
executor.shutdown();
+ try {
+ executor.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ throw new TasteException(e.getCause());
+ }
}
private static Collection<Callable<Void>> wrapWithStatsCallables(Iterable<Callable<Void>> callables,
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegression.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegression.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegression.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegression.java Sun Dec 1 17:42:03 2013
@@ -234,11 +234,12 @@ public class AdaptiveLogisticRegression
return learner.logLikelihood();
}
});
- ep.close();
} catch (InterruptedException e) {
log.warn("Ignoring exception", e);
} catch (ExecutionException e) {
throw new IllegalStateException(e);
+ } finally {
+ ep.close();
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/classify/ClusterClassificationDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/classify/ClusterClassificationDriver.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/classify/ClusterClassificationDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/classify/ClusterClassificationDriver.java Sun Dec 1 17:42:03 2013
@@ -122,15 +122,8 @@ public final class ClusterClassification
* @throws InterruptedException
* @throws ClassNotFoundException
*/
- public static void run(Path input, Path clusteringOutputPath, Path output, Double clusterClassificationThreshold,
+ public static void run(Configuration conf, Path input, Path clusteringOutputPath, Path output, Double clusterClassificationThreshold,
boolean emitMostLikely, boolean runSequential) throws IOException, InterruptedException, ClassNotFoundException {
- Configuration conf = new Configuration();
- run(conf, input, clusteringOutputPath, output, clusterClassificationThreshold, emitMostLikely, runSequential);
- }
-
- public static void run(Configuration conf, Path input, Path clusteringOutputPath, Path output,
- Double clusterClassificationThreshold, boolean emitMostLikely, boolean runSequential)
- throws IOException, InterruptedException, ClassNotFoundException {
if (runSequential) {
classifyClusterSeq(conf, input, clusteringOutputPath, output, clusterClassificationThreshold, emitMostLikely);
} else {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0Mapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0Mapper.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0Mapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0Mapper.java Sun Dec 1 17:42:03 2013
@@ -34,21 +34,21 @@ import java.io.IOException;
* Run ensemble learning via loading the {@link ModelTrainer} with two {@link TopicModel} instances:
* one from the previous iteration, the other empty. Inference is done on the first, and the
* learning updates are stored in the second, and only emitted at cleanup().
- *
+ * <p/>
* In terms of obvious performance improvements still available, the memory footprint in this
* Mapper could be dropped by half if we accumulated model updates onto the model we're using
* for inference, which might also speed up convergence, as we'd be able to take advantage of
* learning <em>during</em> iteration, not just after each one is done. Most likely we don't
* really need to accumulate double values in the model either, floats would most likely be
* sufficient. Between these two, we could squeeze another factor of 4 in memory efficiency.
- *
+ * <p/>
* In terms of CPU, we're re-learning the p(topic|doc) distribution on every iteration, starting
* from scratch. This is usually only 10 fixed-point iterations per doc, but that's 10x more than
* only 1. To avoid having to do this, we would need to do a map-side join of the unchanging
* corpus with the continually-improving p(topic|doc) matrix, and then emit multiple outputs
* from the mappers to make sure we can do the reduce model averaging as well. Tricky, but
* possibly worth it.
- *
+ * <p/>
* {@link ModelTrainer} already takes advantage (in maybe the not-nice way) of multi-core
* availability by doing multithreaded learning, see that class for details.
*/
@@ -58,17 +58,19 @@ public class CachingCVB0Mapper
private static final Logger log = LoggerFactory.getLogger(CachingCVB0Mapper.class);
private ModelTrainer modelTrainer;
+ private TopicModel readModel;
+ private TopicModel writeModel;
private int maxIters;
private int numTopics;
protected ModelTrainer getModelTrainer() {
return modelTrainer;
}
-
+
protected int getMaxIters() {
return maxIters;
}
-
+
protected int getNumTopics() {
return numTopics;
}
@@ -88,7 +90,6 @@ public class CachingCVB0Mapper
float modelWeight = conf.getFloat(CVB0Driver.MODEL_WEIGHT, 1.0f);
log.info("Initializing read model");
- TopicModel readModel;
Path[] modelPaths = CVB0Driver.getModelPaths(conf);
if (modelPaths != null && modelPaths.length > 0) {
readModel = new TopicModel(conf, eta, alpha, null, numUpdateThreads, modelWeight, modelPaths);
@@ -99,7 +100,7 @@ public class CachingCVB0Mapper
}
log.info("Initializing write model");
- TopicModel writeModel = modelWeight == 1
+ writeModel = modelWeight == 1
? new TopicModel(numTopics, numTerms, eta, alpha, null, numUpdateThreads)
: readModel;
@@ -110,7 +111,7 @@ public class CachingCVB0Mapper
@Override
public void map(IntWritable docId, VectorWritable document, Context context)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException {
/* where to get docTopics? */
Vector topicVector = new DenseVector(numTopics).assign(1.0 / numTopics);
modelTrainer.train(document.get(), topicVector, true, maxIters);
@@ -122,9 +123,11 @@ public class CachingCVB0Mapper
modelTrainer.stop();
log.info("Writing model");
- TopicModel model = modelTrainer.getReadModel();
- for (MatrixSlice topic : model) {
+ TopicModel readFrom = modelTrainer.getReadModel();
+ for (MatrixSlice topic : readFrom) {
context.write(new IntWritable(topic.index()), new VectorWritable(topic.vector()));
}
+ readModel.stop();
+ writeModel.stop();
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0PerplexityMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0PerplexityMapper.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0PerplexityMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/CachingCVB0PerplexityMapper.java Sun Dec 1 17:42:03 2013
@@ -44,6 +44,7 @@ public class CachingCVB0PerplexityMapper
private static final Logger log = LoggerFactory.getLogger(CachingCVB0PerplexityMapper.class);
private ModelTrainer modelTrainer;
+ private TopicModel readModel;
private int maxIters;
private int numTopics;
private float testFraction;
@@ -71,7 +72,6 @@ public class CachingCVB0PerplexityMapper
testFraction = conf.getFloat(CVB0Driver.TEST_SET_FRACTION, 0.1f);
log.info("Initializing read model");
- TopicModel readModel;
Path[] modelPaths = CVB0Driver.getModelPaths(conf);
if (modelPaths != null && modelPaths.length > 0) {
readModel = new TopicModel(conf, eta, alpha, null, numUpdateThreads, modelWeight, modelPaths);
@@ -90,6 +90,7 @@ public class CachingCVB0PerplexityMapper
@Override
protected void cleanup(Context context) throws IOException, InterruptedException {
+ readModel.stop();
MemoryUtil.stopMemoryLogger();
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/cvb/TopicModel.java Sun Dec 1 17:42:03 2013
@@ -153,7 +153,15 @@ public class TopicModel implements Confi
return v;
}
- private void initializeThreadPool() {
+ private synchronized void initializeThreadPool() {
+ if (threadPool != null) {
+ threadPool.shutdown();
+ try {
+ threadPool.awaitTermination(100, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ log.error("Could not terminate all threads for TopicModel in time.", e);
+ }
+ }
threadPool = new ThreadPoolExecutor(numThreads, numThreads, 0, TimeUnit.SECONDS,
new ArrayBlockingQueue<Runnable>(numThreads * 10));
threadPool.allowCoreThreadTimeOut(false);
@@ -242,7 +250,7 @@ public class TopicModel implements Confi
return sampler.sample(topicTermCounts.viewRow(topic));
}
- public void reset() {
+ public synchronized void reset() {
for (int x = 0; x < numTopics; x++) {
topicTermCounts.assignRow(x, new SequentialAccessSparseVector(numTerms));
}
@@ -252,7 +260,7 @@ public class TopicModel implements Confi
}
}
- public void stop() {
+ public synchronized void stop() {
for (Updater updater : updaters) {
updater.shutdown();
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/lucene/AnalyzerUtils.java Sun Dec 1 17:42:03 2013
@@ -32,7 +32,7 @@ public final class AnalyzerUtils {
* @throws ClassNotFoundException - {@link ClassNotFoundException}
*/
public static Analyzer createAnalyzer(String analyzerClassName) throws ClassNotFoundException {
- return createAnalyzer(analyzerClassName, Version.LUCENE_43);
+ return createAnalyzer(analyzerClassName, Version.LUCENE_45);
}
public static Analyzer createAnalyzer(String analyzerClassName, Version version) throws ClassNotFoundException {
@@ -47,7 +47,7 @@ public final class AnalyzerUtils {
* @return {@link Analyzer}
*/
public static Analyzer createAnalyzer(Class<? extends Analyzer> analyzerClass) {
- return createAnalyzer(analyzerClass, Version.LUCENE_43);
+ return createAnalyzer(analyzerClass, Version.LUCENE_45);
}
public static Analyzer createAnalyzer(Class<? extends Analyzer> analyzerClass, Version version) {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/ep/EvolutionaryProcess.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/ep/EvolutionaryProcess.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/ep/EvolutionaryProcess.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/ep/EvolutionaryProcess.java Sun Dec 1 17:42:03 2013
@@ -33,6 +33,7 @@ import java.util.concurrent.ExecutionExc
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
/**
* Allows evolutionary optimization where the state function can't be easily
@@ -191,6 +192,11 @@ public class EvolutionaryProcess<T exten
@Override
public void close() {
List<Runnable> remainingTasks = pool.shutdownNow();
+ try {
+ pool.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ throw new IllegalStateException("Had to forcefully shut down " + remainingTasks.size() + " tasks");
+ }
if (!remainingTasks.isEmpty()) {
throw new IllegalStateException("Had to forcefully shut down " + remainingTasks.size() + " tasks");
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ALSWRFactorizerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ALSWRFactorizerTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ALSWRFactorizerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ALSWRFactorizerTest.java Sun Dec 1 17:42:03 2013
@@ -38,6 +38,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+
import java.util.Arrays;
import java.util.Iterator;
@@ -132,6 +134,7 @@ public class ALSWRFactorizerTest extends
}
}
+ @ThreadLeakLingering(linger = 10)
@Test
public void toyExample() throws Exception {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ParallelSGDFactorizerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ParallelSGDFactorizerTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ParallelSGDFactorizerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/recommender/svd/ParallelSGDFactorizerTest.java Sun Dec 1 17:42:03 2013
@@ -20,6 +20,7 @@ package org.apache.mahout.cf.taste.impl.
import java.util.Arrays;
import java.util.List;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import com.google.common.collect.Lists;
import org.apache.mahout.cf.taste.impl.TasteTestCase;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
@@ -184,6 +185,7 @@ public class ParallelSGDFactorizerTest e
assertEquals(index, shuffler.size());
}
+ @ThreadLeakLingering(linger = 1000)
@Test
public void testFactorizerWithToyData() throws Exception {
@@ -240,6 +242,7 @@ public class ParallelSGDFactorizerTest e
assertTrue(rmse < 0.2);
}
+ @ThreadLeakLingering(linger = 1000)
@Test
public void testRecommenderWithToyData() throws Exception {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegressionTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegressionTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegressionTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/AdaptiveLogisticRegressionTest.java Sun Dec 1 17:42:03 2013
@@ -24,10 +24,13 @@ import org.apache.mahout.math.Vector;
import org.apache.mahout.math.jet.random.Exponential;
import org.junit.Test;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
+
import java.util.Random;
public final class AdaptiveLogisticRegressionTest extends MahoutTestCase {
+ @ThreadLeakLingering(linger=1000)
@Test
public void testTrain() {
@@ -49,25 +52,25 @@ public final class AdaptiveLogisticRegre
AdaptiveLogisticRegression.TrainingExample r = getExample(i, gen, beta);
cl.train(r);
if (i % 1000 == 0) {
-// cl.close();
System.out.printf("%10d %10.3f\n", i, cl.getLearner().auc());
}
}
assertEquals(1, cl.getLearner().auc(), 0.1);
- AdaptiveLogisticRegression x = new AdaptiveLogisticRegression(2, 200, new L1());
- x.setInterval(1000);
+ AdaptiveLogisticRegression adaptiveLogisticRegression = new AdaptiveLogisticRegression(2, 200, new L1());
+ adaptiveLogisticRegression.setInterval(1000);
for (int i = 0; i < 20000; i++) {
AdaptiveLogisticRegression.TrainingExample r = getExample(i, gen, beta);
- x.train(r.getKey(), r.getActual(), r.getInstance());
- if (i % 1000 == 0 && x.getBest() != null) {
+ adaptiveLogisticRegression.train(r.getKey(), r.getActual(), r.getInstance());
+ if (i % 1000 == 0 && adaptiveLogisticRegression.getBest() != null) {
System.out.printf("%10d %10.4f %10.8f %.3f\n",
- i, x.auc(),
- Math.log10(x.getBest().getMappedParams()[0]), x.getBest().getMappedParams()[1]);
+ i, adaptiveLogisticRegression.auc(),
+ Math.log10(adaptiveLogisticRegression.getBest().getMappedParams()[0]), adaptiveLogisticRegression.getBest().getMappedParams()[1]);
}
}
- assertEquals(1, x.auc(), 0.1);
+ assertEquals(1, adaptiveLogisticRegression.auc(), 0.1);
+ adaptiveLogisticRegression.close();
}
private static AdaptiveLogisticRegression.TrainingExample getExample(int i, Random gen, Vector beta) {
@@ -146,6 +149,7 @@ public final class AdaptiveLogisticRegre
}
@Test
+ @ThreadLeakLingering(linger = 1000)
public void constantStep() {
AdaptiveLogisticRegression lr = new AdaptiveLogisticRegression(2, 1000, new L1());
lr.setInterval(5000);
@@ -153,10 +157,12 @@ public final class AdaptiveLogisticRegre
assertEquals(20000, lr.nextStep(15001));
assertEquals(20000, lr.nextStep(16500));
assertEquals(20000, lr.nextStep(19999));
+ lr.close();
}
@Test
+ @ThreadLeakLingering(linger = 1000)
public void growingStep() {
AdaptiveLogisticRegression lr = new AdaptiveLogisticRegression(2, 1000, new L1());
lr.setInterval(2000, 10000);
@@ -175,5 +181,6 @@ public final class AdaptiveLogisticRegre
for (int i = 50000; i < 500000; i += 10000) {
assertEquals(i + 10000, lr.nextStep(i));
}
+ lr.close();
}
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/ModelSerializerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/ModelSerializerTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/ModelSerializerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/sgd/ModelSerializerTest.java Sun Dec 1 17:42:03 2013
@@ -24,6 +24,7 @@ import java.io.DataOutputStream;
import java.io.IOException;
import java.util.Random;
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import com.google.common.io.Closeables;
import org.apache.hadoop.io.Writable;
import org.apache.mahout.classifier.OnlineLearner;
@@ -87,6 +88,8 @@ public final class ModelSerializerTest e
train(olr3, 100);
assertEquals(0, olr.getBeta().minus(olr3.getBeta()).aggregate(Functions.MAX, Functions.IDENTITY), 1.0e-6);
+ olr.close();
+ olr3.close();
}
@Test
@@ -107,8 +110,11 @@ public final class ModelSerializerTest e
assertEquals(learner.auc(), olr3.auc(), 0.02);
double auc2 = learner.auc();
assertTrue(auc2 > auc1);
+ learner.close();
+ olr3.close();
}
+ @ThreadLeakLingering(linger = 1000)
@Test
public void adaptiveLogisticRegressionRoundTrip() throws IOException {
AdaptiveLogisticRegression learner = new AdaptiveLogisticRegression(2, 5, new L1());
@@ -128,6 +134,8 @@ public final class ModelSerializerTest e
assertEquals(learner.auc(), olr3.auc(), 0.005);
double auc2 = learner.auc();
assertTrue(String.format("%.3f > %.3f", auc2, auc1), auc2 > auc1);
+ learner.close();
+ olr3.close();
}
private static void train(OnlineLearner olr, int n) {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/ep/EvolutionaryProcessTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/ep/EvolutionaryProcessTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/ep/EvolutionaryProcessTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/ep/EvolutionaryProcessTest.java Sun Dec 1 17:42:03 2013
@@ -52,6 +52,7 @@ public final class EvolutionaryProcessTe
System.out.printf("%10.3f %.3f\n", best.getValue(), best.getOmni());
}
+ ep.close();
assertNotNull(best);
assertEquals(0.0, best.getValue(), 0.02);
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java Sun Dec 1 17:42:03 2013
@@ -106,7 +106,7 @@ public class HighDFWordsPrunerTest exten
String[] args = argList.toArray(new String[argList.size()]);
- ToolRunner.run(getConfiguration(), new SparseVectorsFromSequenceFiles(), args);
+ ToolRunner.run(conf, new SparseVectorsFromSequenceFiles(), args);
Path dictionary = new Path(outputPath, "dictionary.file-0");
Path tfVectors = new Path(outputPath, "tf-vectors");
@@ -134,7 +134,7 @@ public class HighDFWordsPrunerTest exten
}
private void validateVectors(Path vectorPath, int[] highDFWordsDictionaryIndices, boolean prune) throws Exception {
- assertTrue("Path does not exist", vectorPath.getFileSystem(getConfiguration()).exists(vectorPath));
+ assertTrue("Path does not exist", vectorPath.getFileSystem(conf).exists(vectorPath));
for (VectorWritable value : new SequenceFileDirValueIterable<VectorWritable>(vectorPath, PathType.LIST, PathFilters
.partFilter(), null, true, conf)) {
Vector v = ((NamedVector) value.get()).getDelegate();
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/encoders/TextValueEncoderTest.java Sun Dec 1 17:42:03 2013
@@ -70,7 +70,7 @@ public final class TextValueEncoderTest
@Test
public void testLuceneEncoding() throws Exception {
LuceneTextValueEncoder enc = new LuceneTextValueEncoder("text");
- enc.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_43));
+ enc.setAnalyzer(new WhitespaceAnalyzer(Version.LUCENE_45));
Vector v1 = new DenseVector(200);
enc.addToVector("test1 and more", v1);
enc.flush(1, v1);
Modified: mahout/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/pom.xml?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/examples/pom.xml (original)
+++ mahout/trunk/examples/pom.xml Sun Dec 1 17:42:03 2013
@@ -141,15 +141,18 @@
</dependency>
<dependency>
+ <groupId>com.carrotsearch.randomizedtesting</groupId>
+ <artifactId>randomizedtesting-runner</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/NewsgroupHelper.java Sun Dec 1 17:42:03 2013
@@ -60,7 +60,7 @@ public final class NewsgroupHelper {
private static final long WEEK = 7 * 24 * 3600;
private final Random rand = RandomUtils.getRandom();
- private final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_43);
+ private final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_45);
private final FeatureVectorEncoder encoder = new StaticWordValueEncoder("body");
private final FeatureVectorEncoder bias = new ConstantValueEncoder("Intercept");
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneStorageConfiguration.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneStorageConfiguration.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneStorageConfiguration.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/LuceneStorageConfiguration.java Sun Dec 1 17:42:03 2013
@@ -44,7 +44,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import static org.apache.lucene.util.Version.LUCENE_43;
+import static org.apache.lucene.util.Version.LUCENE_45;
/**
* Holds all the configuration for {@link SequenceFilesFromLuceneStorage}, which generates a sequence file
@@ -212,7 +212,7 @@ public class LuceneStorageConfiguration
}
idField = in.readUTF();
fields = Arrays.asList(in.readUTF().split(SEPARATOR_FIELDS));
- query = new QueryParser(LUCENE_43, "query", new StandardAnalyzer(LUCENE_43)).parse(in.readUTF());
+ query = new QueryParser(LUCENE_45, "query", new StandardAnalyzer(LUCENE_45)).parse(in.readUTF());
maxHits = in.readInt();
} catch (ParseException e) {
throw new RuntimeException("Could not deserialize " + this.getClass().getName(), e);
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/MailArchivesClusteringAnalyzer.java Sun Dec 1 17:42:03 2013
@@ -41,7 +41,7 @@ import org.apache.lucene.util.Version;
* stop words, excluding non-alpha-numeric tokens, and porter stemming.
*/
public final class MailArchivesClusteringAnalyzer extends StopwordAnalyzerBase {
- private static final Version LUCENE_VERSION = Version.LUCENE_43;
+ private static final Version LUCENE_VERSION = Version.LUCENE_45;
// extended set of stop words composed of common mail terms like "hi",
// HTML tags, and Java keywords asmany of the messages in the archives
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriver.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriver.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/SequenceFilesFromLuceneStorageDriver.java Sun Dec 1 17:42:03 2013
@@ -97,8 +97,8 @@ public class SequenceFilesFromLuceneStor
if (hasOption(OPTION_QUERY)) {
try {
String queryString = COMPILE.matcher(getOption(OPTION_QUERY)).replaceAll("");
- QueryParser queryParser = new QueryParser(Version.LUCENE_43, queryString,
- new StandardAnalyzer(Version.LUCENE_43));
+ QueryParser queryParser = new QueryParser(Version.LUCENE_45, queryString,
+ new StandardAnalyzer(Version.LUCENE_45));
query = queryParser.parse(queryString);
} catch (ParseException e) {
throw new IllegalArgumentException(e.getMessage(), e);
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/text/wikipedia/WikipediaAnalyzer.java Sun Dec 1 17:42:03 2013
@@ -33,19 +33,19 @@ import org.apache.lucene.util.Version;
public class WikipediaAnalyzer extends StopwordAnalyzerBase {
public WikipediaAnalyzer() {
- super(Version.LUCENE_43, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
+ super(Version.LUCENE_45, StopAnalyzer.ENGLISH_STOP_WORDS_SET);
}
public WikipediaAnalyzer(CharArraySet stopSet) {
- super(Version.LUCENE_43, stopSet);
+ super(Version.LUCENE_45, stopSet);
}
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new WikipediaTokenizer(reader);
- TokenStream result = new StandardFilter(Version.LUCENE_43, tokenizer);
- result = new LowerCaseFilter(Version.LUCENE_43, result);
- result = new StopFilter(Version.LUCENE_43, result, getStopwordSet());
+ TokenStream result = new StandardFilter(Version.LUCENE_45, tokenizer);
+ result = new LowerCaseFilter(Version.LUCENE_45, result);
+ result = new StopFilter(Version.LUCENE_45, result, getStopwordSet());
return new TokenStreamComponents(tokenizer, result);
}
}
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/regex/AnalyzerTransformer.java Sun Dec 1 17:42:03 2013
@@ -38,7 +38,7 @@ public class AnalyzerTransformer impleme
private static final Logger log = LoggerFactory.getLogger(AnalyzerTransformer.class);
public AnalyzerTransformer() {
- this(new StandardAnalyzer(Version.LUCENE_43), "text");
+ this(new StandardAnalyzer(Version.LUCENE_45), "text");
}
public AnalyzerTransformer(Analyzer analyzer) {
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java Sun Dec 1 17:42:03 2013
@@ -97,7 +97,7 @@ public final class TestClusterDumper ext
RAMDirectory directory = new RAMDirectory();
IndexWriter writer = new IndexWriter(directory,
- new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43)));
+ new IndexWriterConfig(Version.LUCENE_45, new StandardAnalyzer(Version.LUCENE_45)));
try {
for (int i = 0; i < docs2.length; i++) {
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/AbstractLuceneStorageTest.java Sun Dec 1 17:42:03 2013
@@ -60,7 +60,7 @@ public abstract class AbstractLuceneStor
}
protected void commitDocuments(Directory directory, Iterable<SingleFieldDocument> theDocs) throws IOException{
- IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43)));
+ IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_45, new StandardAnalyzer(Version.LUCENE_45)));
for (SingleFieldDocument singleFieldDocument : theDocs) {
indexWriter.addDocument(singleFieldDocument.asLuceneDocument());
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/nlp/collocations/llr/BloomTokenFilterTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/nlp/collocations/llr/BloomTokenFilterTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/nlp/collocations/llr/BloomTokenFilterTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/nlp/collocations/llr/BloomTokenFilterTest.java Sun Dec 1 17:42:03 2013
@@ -80,7 +80,7 @@ public final class BloomTokenFilterTest
@Test
public void testAnalyzer() throws IOException {
Reader reader = new StringReader(input);
- Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_43);
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_45);
TokenStream ts = analyzer.tokenStream(null, reader);
ts.reset();
validateTokens(allTokens, ts);
@@ -92,7 +92,7 @@ public final class BloomTokenFilterTest
@Test
public void testNonKeepdAnalyzer() throws IOException {
Reader reader = new StringReader(input);
- Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_43);
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_45);
TokenStream ts = analyzer.tokenStream(null, reader);
ts.reset();
TokenStream f = new BloomTokenFilter(getFilter(filterTokens), false /* toss matching tokens */, ts);
@@ -105,7 +105,7 @@ public final class BloomTokenFilterTest
@Test
public void testKeepAnalyzer() throws IOException {
Reader reader = new StringReader(input);
- Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_43);
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_45);
TokenStream ts = analyzer.tokenStream(null, reader);
ts.reset();
TokenStream f = new BloomTokenFilter(getFilter(filterTokens), true /* keep matching tokens */, ts);
@@ -118,7 +118,7 @@ public final class BloomTokenFilterTest
@Test
public void testShingleFilteredAnalyzer() throws IOException {
Reader reader = new StringReader(input);
- Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_43);
+ Analyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_45);
TokenStream ts = analyzer.tokenStream(null, reader);
ts.reset();
ShingleFilter sf = new ShingleFilter(ts, 3);
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfoTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfoTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfoTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/CachedTermInfoTest.java Sun Dec 1 17:42:03 2013
@@ -100,7 +100,7 @@ public class CachedTermInfoTest extends
static RAMDirectory createTestIndex(FieldType fieldType,
RAMDirectory directory,
int startingId) throws IOException {
- IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43)));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_45, new WhitespaceAnalyzer(Version.LUCENE_45)));
try {
for (int i = 0; i < DOCS.length; i++) {
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/DriverTest.java Sun Dec 1 17:42:03 2013
@@ -91,8 +91,8 @@ public class DriverTest extends MahoutTe
public void sequenceFileDictionary() throws IOException {
Directory index = new SimpleFSDirectory(indexDir);
- Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_43);
- IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, analyzer);
+ Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_45);
+ IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_45, analyzer);
final IndexWriter writer = new IndexWriter(index, config);
try {
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/utils/vectors/lucene/LuceneIterableTest.java Sun Dec 1 17:42:03 2013
@@ -185,7 +185,7 @@ public final class LuceneIterableTest ex
static RAMDirectory createTestIndex(FieldType fieldType,
RAMDirectory directory,
int startingId) throws IOException {
- IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_43,new StandardAnalyzer(Version.LUCENE_43)));
+ IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Version.LUCENE_45,new StandardAnalyzer(Version.LUCENE_45)));
try {
for (int i = 0; i < DOCS.length; i++) {
Modified: mahout/trunk/math/src/main/java/org/apache/mahout/math/decomposer/AsyncEigenVerifier.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/main/java/org/apache/mahout/math/decomposer/AsyncEigenVerifier.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/math/src/main/java/org/apache/mahout/math/decomposer/AsyncEigenVerifier.java (original)
+++ mahout/trunk/math/src/main/java/org/apache/mahout/math/decomposer/AsyncEigenVerifier.java Sun Dec 1 17:42:03 2013
@@ -17,16 +17,16 @@
package org.apache.mahout.math.decomposer;
-import java.util.concurrent.Executor;
+import java.io.Closeable;
+import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorIterable;
+public class AsyncEigenVerifier extends SimpleEigenVerifier implements Closeable {
-public class AsyncEigenVerifier extends SimpleEigenVerifier {
-
- private final Executor threadPool;
+ private final ExecutorService threadPool;
private EigenStatus status;
private boolean finished;
private boolean started;
@@ -50,6 +50,10 @@ public class AsyncEigenVerifier extends
return status;
}
+ @Override
+ public void close() {
+ this.threadPool.shutdownNow();
+ }
protected EigenStatus innerVerify(VectorIterable corpus, Vector vector) {
return super.verify(corpus, vector);
}
Modified: mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java (original)
+++ mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java Sun Dec 1 17:42:03 2013
@@ -23,13 +23,14 @@ import java.io.IOException;
import org.apache.mahout.common.RandomUtils;
import org.junit.After;
-import org.junit.Assert;
import org.junit.Before;
+import com.carrotsearch.randomizedtesting.RandomizedTest;
+
/**
* Superclass of all Mahout test cases.
*/
-public abstract class MahoutTestCase extends Assert {
+public abstract class MahoutTestCase extends RandomizedTest {
/** "Close enough" value for floating-point comparisons. */
public static final double EPSILON = 0.000001;
Modified: mahout/trunk/math/src/test/java/org/apache/mahout/math/decomposer/hebbian/TestHebbianSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/test/java/org/apache/mahout/math/decomposer/hebbian/TestHebbianSolver.java?rev=1546825&r1=1546824&r2=1546825&view=diff
==============================================================================
--- mahout/trunk/math/src/test/java/org/apache/mahout/math/decomposer/hebbian/TestHebbianSolver.java (original)
+++ mahout/trunk/math/src/test/java/org/apache/mahout/math/decomposer/hebbian/TestHebbianSolver.java Sun Dec 1 17:42:03 2013
@@ -21,7 +21,6 @@ import org.apache.mahout.math.DenseMatri
import org.apache.mahout.math.Matrix;
import org.apache.mahout.math.decomposer.AsyncEigenVerifier;
-import org.apache.mahout.math.decomposer.SingularVectorVerifier;
import org.apache.mahout.math.decomposer.SolverTest;
import org.junit.Test;
@@ -50,7 +49,7 @@ public final class TestHebbianSolver ext
int desiredRank,
TrainingState state) {
HebbianUpdater updater = new HebbianUpdater();
- SingularVectorVerifier verifier = new AsyncEigenVerifier();
+ AsyncEigenVerifier verifier = new AsyncEigenVerifier();
HebbianSolver solver = new HebbianSolver(updater,
verifier,
convergence,
@@ -62,6 +61,7 @@ public final class TestHebbianSolver ext
state.setCurrentEigenValues(finalState.getCurrentEigenValues());
long time = 0L;
time += System.nanoTime() - start;
+ verifier.close();
assertEquals(state.getCurrentEigens().numRows(), desiredRank);
return time / 1000000L;
}