You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by is...@apache.org on 2013/06/02 00:24:04 UTC
svn commit: r1488627 [1/2] - in /mahout/trunk: ./
core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/
core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/
core/src/test/java/org/apache/mahout/classifier/naivebayes/ core/src/test...
Author: isabel
Date: Sat Jun 1 22:24:03 2013
New Revision: 1488627
URL: http://svn.apache.org/r1488627
Log:
MAHOUT-1200 - Mahout tests depend on writing to /tmp/hadoop-$user
Change hadoop dependent tests to work out of a directory that is not under /tmp
Modified:
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/classify/ClusterClassificationDriverTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDirichletClustering.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestMapReduce.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/iterator/TestClusterClassifier.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/lda/cvb/TestCVBModelTrainer.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java
mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java
mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/FPGrowthTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthRetailDataTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolver.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolverCLI.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestVectorDistanceSimilarityJob.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/cooccurrence/RowSimilarityJobTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolver.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolverCLI.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/stats/BasicStatsTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/ConditionalEntropyTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/EntropyTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainRatioTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/EncodedVectorsFromSequenceFilesTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java
mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFilesTest.java
mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SplitInput.java
mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java
mahout/trunk/integration/src/test/java/org/apache/mahout/utils/SplitInputTest.java
mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java
mahout/trunk/pom.xml
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJobTest.java Sat Jun 1 22:24:03 2013
@@ -724,7 +724,7 @@ public class RecommenderJobTest extends
RecommenderJob recommenderJob = new RecommenderJob();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set("mapred.input.dir", inputFile.getAbsolutePath());
conf.set("mapred.output.dir", outputDir.getAbsolutePath());
conf.setBoolean("mapred.output.compress", false);
@@ -810,7 +810,7 @@ public class RecommenderJobTest extends
RecommenderJob recommenderJob = new RecommenderJob();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set("mapred.input.dir", inputFile.getAbsolutePath());
conf.set("mapred.output.dir", outputDir.getAbsolutePath());
conf.setBoolean("mapred.output.compress", false);
@@ -871,7 +871,7 @@ public class RecommenderJobTest extends
RecommenderJob recommenderJob = new RecommenderJob();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set("mapred.input.dir", inputFile.getAbsolutePath());
conf.set("mapred.output.dir", outputDir.getAbsolutePath());
conf.setBoolean("mapred.output.compress", false);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJobTest.java Sat Jun 1 22:24:03 2013
@@ -127,16 +127,14 @@ public final class ItemSimilarityJobTest
ItemSimilarityJob similarityJob = new ItemSimilarityJob();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set("mapred.input.dir", inputFile.getAbsolutePath());
conf.set("mapred.output.dir", outputDir.getAbsolutePath());
conf.setBoolean("mapred.output.compress", false);
similarityJob.setConf(conf);
-
similarityJob.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--similarityClassname",
CosineSimilarity.class.getName() });
-
File outPart = outputDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
@@ -223,16 +221,14 @@ public final class ItemSimilarityJobTest
ItemSimilarityJob similarityJob = new ItemSimilarityJob();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set("mapred.input.dir", inputFile.getAbsolutePath());
conf.set("mapred.output.dir", outputDir.getAbsolutePath());
conf.setBoolean("mapred.output.compress", false);
similarityJob.setConf(conf);
-
similarityJob.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--similarityClassname",
TanimotoCoefficientSimilarity.class.getName(), "--maxSimilaritiesPerItem", "1" });
-
File outPart = outputDir.listFiles(new FilenameFilter() {
@Override
public boolean accept(File dir, String name) {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/classifier/naivebayes/NaiveBayesTest.java Sat Jun 1 22:24:03 2013
@@ -58,7 +58,7 @@ public class NaiveBayesTest extends Maho
public void setUp() throws Exception {
super.setUp();
- conf = new Configuration();
+ conf = getConfiguration();
inputFile = getTestTempFile("trainingInstances.seq");
outputDir = getTestTempDir("output");
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java Sat Jun 1 22:24:03 2013
@@ -106,7 +106,7 @@ public final class TestCanopyCreation ex
@Before
public void setUp() throws Exception {
super.setUp();
- fs = FileSystem.get(new Configuration());
+ fs = FileSystem.get(getConfiguration());
referenceManhattan = CanopyClusterer.createCanopies(getPoints(),
manhattanDistanceMeasure, 3.1, 2.1);
manhattanCentroids = CanopyClusterer.getCenters(referenceManhattan);
@@ -173,7 +173,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyMapperManhattan() throws Exception {
CanopyMapper mapper = new CanopyMapper();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, manhattanDistanceMeasure
.getClass().getName());
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
@@ -209,7 +209,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyMapperEuclidean() throws Exception {
CanopyMapper mapper = new CanopyMapper();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, euclideanDistanceMeasure
.getClass().getName());
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
@@ -245,7 +245,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyReducerManhattan() throws Exception {
CanopyReducer reducer = new CanopyReducer();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY,
"org.apache.mahout.common.distance.ManhattanDistanceMeasure");
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
@@ -281,7 +281,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyReducerEuclidean() throws Exception {
CanopyReducer reducer = new CanopyReducer();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY,
"org.apache.mahout.common.distance.EuclideanDistanceMeasure");
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
@@ -316,7 +316,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyGenManhattanMR() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration config = new Configuration();
+ Configuration config = getConfiguration();
ClusteringTestUtils.writePointsToFile(points,
getTestTempFilePath("testdata/file1"), fs, config);
ClusteringTestUtils.writePointsToFile(points,
@@ -373,7 +373,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyGenEuclideanMR() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration config = new Configuration();
+ Configuration config = getConfiguration();
ClusteringTestUtils.writePointsToFile(points,
getTestTempFilePath("testdata/file1"), fs, config);
ClusteringTestUtils.writePointsToFile(points,
@@ -414,7 +414,7 @@ public final class TestCanopyCreation ex
@Test
public void testClusteringManhattanSeq() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration config = new Configuration();
+ Configuration config = getConfiguration();
ClusteringTestUtils.writePointsToFile(points,
getTestTempFilePath("testdata/file1"), fs, config);
// now run the Canopy Driver in sequential mode
@@ -441,7 +441,7 @@ public final class TestCanopyCreation ex
@Test
public void testClusteringEuclideanSeq() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration config = new Configuration();
+ Configuration config = getConfiguration();
ClusteringTestUtils.writePointsToFile(points,
getTestTempFilePath("testdata/file1"), fs, config);
// now run the Canopy Driver in sequential mode
@@ -457,7 +457,7 @@ public final class TestCanopyCreation ex
optKey(DefaultOptionCreator.OVERWRITE_OPTION),
optKey(DefaultOptionCreator.METHOD_OPTION),
DefaultOptionCreator.SEQUENTIAL_METHOD };
- new CanopyDriver().run(args);
+ ToolRunner.run(config, new CanopyDriver(), args);
// verify output from sequence file
Path path = new Path(output, "clusters-0-final/part-r-00000");
@@ -479,7 +479,7 @@ public final class TestCanopyCreation ex
@Test
public void testClusteringEuclideanWithOutlierRemovalSeq() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration config = new Configuration();
+ Configuration config = getConfiguration();
ClusteringTestUtils.writePointsToFile(points,
getTestTempFilePath("testdata/file1"), fs, config);
// now run the Canopy Driver in sequential mode
@@ -496,7 +496,7 @@ public final class TestCanopyCreation ex
optKey(DefaultOptionCreator.OVERWRITE_OPTION),
optKey(DefaultOptionCreator.METHOD_OPTION),
DefaultOptionCreator.SEQUENTIAL_METHOD };
- new CanopyDriver().run(args);
+ ToolRunner.run(config, new CanopyDriver(), args);
// verify output from sequence file
Path path = new Path(output, "clusters-0-final/part-r-00000");
@@ -523,7 +523,7 @@ public final class TestCanopyCreation ex
@Test
public void testClusteringManhattanMR() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true,
getTestTempFilePath("testdata/file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(points, true,
@@ -544,7 +544,7 @@ public final class TestCanopyCreation ex
@Test
public void testClusteringEuclideanMR() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true,
getTestTempFilePath("testdata/file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(points, true,
@@ -560,7 +560,7 @@ public final class TestCanopyCreation ex
optKey(DefaultOptionCreator.T2_OPTION), "2.1",
optKey(DefaultOptionCreator.CLUSTERING_OPTION),
optKey(DefaultOptionCreator.OVERWRITE_OPTION) };
- ToolRunner.run(new Configuration(), new CanopyDriver(), args);
+ ToolRunner.run(getConfiguration(), new CanopyDriver(), args);
Path path = new Path(output, "clusteredPoints/part-m-00000");
long count = HadoopUtil.countRecords(path, conf);
assertEquals("number of points", points.size(), count);
@@ -573,7 +573,7 @@ public final class TestCanopyCreation ex
@Test
public void testClusteringEuclideanWithOutlierRemovalMR() throws Exception {
List<VectorWritable> points = getPointsWritable();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true,
getTestTempFilePath("testdata/file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(points, true,
@@ -590,7 +590,7 @@ public final class TestCanopyCreation ex
optKey(DefaultOptionCreator.OUTLIER_THRESHOLD), "0.7",
optKey(DefaultOptionCreator.CLUSTERING_OPTION),
optKey(DefaultOptionCreator.OVERWRITE_OPTION) };
- ToolRunner.run(new Configuration(), new CanopyDriver(), args);
+ ToolRunner.run(getConfiguration(), new CanopyDriver(), args);
Path path = new Path(output, "clusteredPoints/part-m-00000");
long count = HadoopUtil.countRecords(path, conf);
int expectedPointsAfterOutlierRemoval = 8;
@@ -605,7 +605,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyReducerT3T4Configuration() throws Exception {
CanopyReducer reducer = new CanopyReducer();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY,
"org.apache.mahout.common.distance.ManhattanDistanceMeasure");
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
@@ -628,7 +628,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyMapperClusterFilter() throws Exception {
CanopyMapper mapper = new CanopyMapper();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, manhattanDistanceMeasure
.getClass().getName());
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
@@ -658,7 +658,7 @@ public final class TestCanopyCreation ex
@Test
public void testCanopyReducerClusterFilter() throws Exception {
CanopyReducer reducer = new CanopyReducer();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY,
"org.apache.mahout.common.distance.ManhattanDistanceMeasure");
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(3.1));
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/classify/ClusterClassificationDriverTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/classify/ClusterClassificationDriverTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/classify/ClusterClassificationDriverTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/classify/ClusterClassificationDriverTest.java Sat Jun 1 22:24:03 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.clustering.ClusteringTestUtils;
import org.apache.mahout.clustering.canopy.CanopyDriver;
import org.apache.mahout.clustering.iterator.CanopyClusteringPolicy;
@@ -65,7 +66,7 @@ public class ClusterClassificationDriver
@Before
public void setUp() throws Exception {
super.setUp();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
fs = FileSystem.get(conf);
firstCluster = Lists.newArrayList();
secondCluster = Lists.newArrayList();
@@ -92,7 +93,7 @@ public class ClusterClassificationDriver
classifiedOutputPath = getTestTempDirPath("classifiedClusters");
HadoopUtil.delete(conf, classifiedOutputPath);
- conf = new Configuration();
+ conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true,
new Path(pointsPath, "file1"), fs, conf);
@@ -110,7 +111,7 @@ public class ClusterClassificationDriver
clusteringOutputPath = getTestTempDirPath("output");
classifiedOutputPath = getTestTempDirPath("classify");
- conf = new Configuration();
+ conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points,
new Path(pointsPath, "file1"), fs, conf);
@@ -150,12 +151,12 @@ public class ClusterClassificationDriver
private void runClassificationWithoutOutlierRemoval()
throws IOException, InterruptedException, ClassNotFoundException {
- ClusterClassificationDriver.run(pointsPath, clusteringOutputPath, classifiedOutputPath, 0.0, true, true);
+ ClusterClassificationDriver.run(getConfiguration(), pointsPath, clusteringOutputPath, classifiedOutputPath, 0.0, true, true);
}
private void runClassificationWithOutlierRemoval(boolean runSequential)
throws IOException, InterruptedException, ClassNotFoundException {
- ClusterClassificationDriver.run(pointsPath, clusteringOutputPath, classifiedOutputPath, 0.73, true, runSequential);
+ ClusterClassificationDriver.run(getConfiguration(), pointsPath, clusteringOutputPath, classifiedOutputPath, 0.73, true, runSequential);
}
private void collectVectorsForAssertion() throws IOException {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDirichletClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDirichletClustering.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDirichletClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestDirichletClustering.java Sat Jun 1 22:24:03 2013
@@ -81,8 +81,8 @@ public final class TestDirichletClusteri
@Test
public void testDirichletClusteringSeq() throws Exception {
Path output = getTestTempDirPath("output");
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(new Configuration());
+ Configuration conf = getConfiguration();
+ FileSystem fs = FileSystem.get(getConfiguration());
generateSamples(40, 1, 1, 3);
generateSamples(30, 1, 0, 0.1);
@@ -109,8 +109,8 @@ public final class TestDirichletClusteri
@Test
public void testDirichletClusteringMR() throws Exception {
Path output = getTestTempDirPath("output");
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(new Configuration());
+ Configuration conf = getConfiguration();
+ FileSystem fs = FileSystem.get(getConfiguration());
generateSamples(40, 1, 1, 3);
generateSamples(30, 1, 0, 0.1);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestMapReduce.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestMapReduce.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestMapReduce.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestMapReduce.java Sat Jun 1 22:24:03 2013
@@ -107,7 +107,7 @@ public final class TestMapReduce extends
@Before
public void setUp() throws Exception {
super.setUp();
- conf = new Configuration();
+ conf = getConfiguration();
fs = FileSystem.get(conf);
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java Sat Jun 1 22:24:03 2013
@@ -48,7 +48,7 @@ public final class TestFuzzyKmeansCluste
@Before
public void setUp() throws Exception {
super.setUp();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
fs = FileSystem.get(conf);
}
@@ -62,7 +62,7 @@ public final class TestFuzzyKmeansCluste
Path pointsPath = getTestTempDirPath("points");
Path clustersPath = getTestTempDirPath("clusters");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, new Path(pointsPath, "file1"), fs, conf);
for (int k = 0; k < points.size(); k++) {
@@ -133,7 +133,7 @@ public final class TestFuzzyKmeansCluste
Path pointsPath = getTestTempDirPath("points");
Path clustersPath = getTestTempDirPath("clusters");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, new Path(pointsPath, "file1"), fs, conf);
for (int k = 0; k < points.size(); k++) {
@@ -192,7 +192,7 @@ public final class TestFuzzyKmeansCluste
optKey(DefaultOptionCreator.EMIT_MOST_LIKELY_OPTION),
optKey(DefaultOptionCreator.OVERWRITE_OPTION)
};
- ToolRunner.run(new Configuration(), new FuzzyKMeansDriver(), args);
+ ToolRunner.run(getConfiguration(), new FuzzyKMeansDriver(), args);
long count = HadoopUtil.countRecords(new Path(output, "clusteredPoints/part-m-00000"), conf);
assertTrue(count > 0);
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/iterator/TestClusterClassifier.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/iterator/TestClusterClassifier.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/iterator/TestClusterClassifier.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/iterator/TestClusterClassifier.java Sat Jun 1 22:24:03 2013
@@ -93,7 +93,7 @@ public final class TestClusterClassifier
Path path = new Path(getTestTempDirPath(), "output");
classifier.writeToSeqFiles(path);
ClusterClassifier newClassifier = new ClusterClassifier();
- newClassifier.readFromSeqFiles(new Configuration(), path);
+ newClassifier.readFromSeqFiles(getConfiguration(), path);
return newClassifier;
}
@@ -221,7 +221,7 @@ public final class TestClusterClassifier
Path pointsPath = getTestTempDirPath("points");
Path priorPath = getTestTempDirPath("prior");
Path outPath = getTestTempDirPath("output");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(pointsPath.toUri(), conf);
List<VectorWritable> points = TestKmeansClustering.getPointsWritable(TestKmeansClustering.REFERENCE);
ClusteringTestUtils.writePointsToFile(points, new Path(pointsPath, "file1"), fs, conf);
@@ -253,7 +253,7 @@ public final class TestClusterClassifier
Path pointsPath = getTestTempDirPath("points");
Path priorPath = getTestTempDirPath("prior");
Path outPath = getTestTempDirPath("output");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(pointsPath.toUri(), conf);
List<VectorWritable> points = TestKmeansClustering.getPointsWritable(TestKmeansClustering.REFERENCE);
ClusteringTestUtils.writePointsToFile(points, new Path(pointsPath, "file1"), fs, conf);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java Sat Jun 1 22:24:03 2013
@@ -62,7 +62,7 @@ public final class TestKmeansClustering
@Before
public void setUp() throws Exception {
super.setUp();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
fs = FileSystem.get(conf);
}
@@ -127,7 +127,7 @@ public final class TestKmeansClustering
Path pointsPath = getTestTempDirPath("points");
Path clustersPath = getTestTempDirPath("clusters");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(pointsPath, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(points, true, new Path(pointsPath, "file2"), fs, conf);
for (int k = 1; k < points.size(); k++) {
@@ -160,7 +160,7 @@ public final class TestKmeansClustering
optKey(DefaultOptionCreator.MAX_ITERATIONS_OPTION), "2", optKey(DefaultOptionCreator.CLUSTERING_OPTION),
optKey(DefaultOptionCreator.OVERWRITE_OPTION), optKey(DefaultOptionCreator.METHOD_OPTION),
DefaultOptionCreator.SEQUENTIAL_METHOD};
- new KMeansDriver().run(args);
+ ToolRunner.run(conf, new KMeansDriver(), args);
// now compare the expected clusters with actual
Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
@@ -183,7 +183,7 @@ public final class TestKmeansClustering
Path pointsPath = getTestTempDirPath("points");
Path clustersPath = getTestTempDirPath("clusters");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(pointsPath, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(points, true, new Path(pointsPath, "file2"), fs, conf);
for (int k = 1; k < points.size(); k += 3) {
@@ -216,7 +216,7 @@ public final class TestKmeansClustering
optKey(DefaultOptionCreator.CONVERGENCE_DELTA_OPTION), "0.001",
optKey(DefaultOptionCreator.MAX_ITERATIONS_OPTION), "2", optKey(DefaultOptionCreator.CLUSTERING_OPTION),
optKey(DefaultOptionCreator.OVERWRITE_OPTION)};
- ToolRunner.run(new Configuration(), new KMeansDriver(), args);
+ ToolRunner.run(getConfiguration(), new KMeansDriver(), args);
// now compare the expected clusters with actual
Path clusteredPointsPath = new Path(outputPath, "clusteredPoints");
@@ -241,7 +241,7 @@ public final class TestKmeansClustering
List<VectorWritable> points = getPointsWritable(REFERENCE);
Path pointsPath = getTestTempDirPath("points");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(pointsPath, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(points, true, new Path(pointsPath, "file2"), fs, conf);
@@ -288,7 +288,7 @@ public final class TestKmeansClustering
// now run the KMeans job
Path kmeansOutput = new Path(outputPath, "kmeans");
- KMeansDriver.run(pointsPath, new Path(outputPath, "clusters-0-final"), kmeansOutput, new EuclideanDistanceMeasure(),
+ KMeansDriver.run(getConfiguration(), pointsPath, new Path(outputPath, "clusters-0-final"), kmeansOutput, new EuclideanDistanceMeasure(),
0.001, 10, true, 0.0, false);
// now compare the expected clusters with actual
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/lda/cvb/TestCVBModelTrainer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/lda/cvb/TestCVBModelTrainer.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/lda/cvb/TestCVBModelTrainer.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/lda/cvb/TestCVBModelTrainer.java Sat Jun 1 22:24:03 2013
@@ -95,14 +95,15 @@ public final class TestCVBModelTrainer e
numDocs, numSamples, numTopicsPerDoc);
Path sampleCorpusPath = getTestTempDirPath("corpus");
- MatrixUtils.write(sampleCorpusPath, new Configuration(), sampledCorpus);
+ Configuration configuration = getConfiguration();
+ MatrixUtils.write(sampleCorpusPath, configuration, sampledCorpus);
int numIterations = 5;
List<Double> perplexities = Lists.newArrayList();
int startTopic = numGeneratingTopics - 1;
int numTestTopics = startTopic;
while (numTestTopics < numGeneratingTopics + 2) {
Path topicModelStateTempPath = getTestTempDirPath("topicTemp" + numTestTopics);
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
CVB0Driver.run(conf, sampleCorpusPath, null, numTestTopics, numTerms,
ALPHA, ETA, numIterations, 1, 0, null, null, topicModelStateTempPath, 1234, 0.2f, 2,
1, 3, 1, false);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java Sat Jun 1 22:24:03 2013
@@ -193,7 +193,7 @@ public final class TestMeanShift extends
euclideanDistanceMeasure), refCanopies);
}
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(MeanShiftCanopyConfigKeys.DISTANCE_MEASURE_KEY, EuclideanDistanceMeasure.class.getName());
conf.set(MeanShiftCanopyConfigKeys.KERNEL_PROFILE_KEY, TriangularKernelProfile.class.getName());
conf.set(MeanShiftCanopyConfigKeys.T1_KEY, "4");
@@ -278,7 +278,7 @@ public final class TestMeanShift extends
clusterer.shiftToMean(canopy);
}
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
conf.set(MeanShiftCanopyConfigKeys.DISTANCE_MEASURE_KEY, EuclideanDistanceMeasure.class.getName());
conf.set(MeanShiftCanopyConfigKeys.KERNEL_PROFILE_KEY, TriangularKernelProfile.class.getName());
conf.set(MeanShiftCanopyConfigKeys.T1_KEY, "4");
@@ -353,7 +353,7 @@ public final class TestMeanShift extends
@Test
public void testCanopyEuclideanMRJob() throws Exception {
Path input = getTestTempDirPath("testdata");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(input.toUri(), conf);
Collection<VectorWritable> points = Lists.newArrayList();
// TODO fix test so it doesn't need this random seed!
@@ -414,7 +414,7 @@ public final class TestMeanShift extends
@Test
public void testCanopyEuclideanSeqJob() throws Exception {
Path input = getTestTempDirPath("testdata");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(input.toUri(), conf);
Collection<VectorWritable> points = Lists.newArrayList();
for (Vector v : raw) {
@@ -445,7 +445,7 @@ public final class TestMeanShift extends
optKey(DefaultOptionCreator.OVERWRITE_OPTION),
optKey(DefaultOptionCreator.METHOD_OPTION),
DefaultOptionCreator.SEQUENTIAL_METHOD };
- ToolRunner.run(new Configuration(), new MeanShiftCanopyDriver(), args);
+ ToolRunner.run(getConfiguration(), new MeanShiftCanopyDriver(), args);
Path outPart = new Path(output, "clusters-7-final/part-r-00000");
long count = HadoopUtil.countRecords(outPart, conf);
assertEquals("count", 3, count);
@@ -458,7 +458,7 @@ public final class TestMeanShift extends
@Test
public void testCanopyEuclideanMRJobNoClustering() throws Exception {
Path input = getTestTempDirPath("testdata");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(input.toUri(), conf);
Collection<VectorWritable> points = Lists.newArrayList();
for (Vector v : raw) {
@@ -506,7 +506,7 @@ public final class TestMeanShift extends
@Test
public void testCanopyEuclideanSeqJobNoClustering() throws Exception {
Path input = getTestTempDirPath("testdata");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(input.toUri(), conf);
Collection<VectorWritable> points = Lists.newArrayList();
for (Vector v : raw) {
@@ -536,7 +536,7 @@ public final class TestMeanShift extends
optKey(DefaultOptionCreator.OVERWRITE_OPTION),
optKey(DefaultOptionCreator.METHOD_OPTION),
DefaultOptionCreator.SEQUENTIAL_METHOD };
- ToolRunner.run(new Configuration(), new MeanShiftCanopyDriver(), args);
+ ToolRunner.run(getConfiguration(), new MeanShiftCanopyDriver(), args);
Path outPart = new Path(output, "clusters-7-final/part-r-00000");
long count = HadoopUtil.countRecords(outPart, conf);
assertEquals("count", 3, count);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/clustering/minhash/TestMinHashClustering.java Sat Jun 1 22:24:03 2013
@@ -36,6 +36,7 @@ import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.junit.Test;
+import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@@ -63,7 +64,7 @@ public final class TestMinHashClustering
@Override
public void setUp() throws Exception {
super.setUp();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
List<VectorWritable> points = getPointsWritable(REFERENCE);
input = getTestTempDirPath("points");
output = new Path(getTestTempDirPath(), "output");
@@ -124,8 +125,8 @@ public final class TestMinHashClustering
}
}
- private static void verify(Path output, double simThreshold, String msg) {
- Configuration conf = new Configuration();
+ private void verify(Path output, double simThreshold, String msg) throws IOException {
+ Configuration conf = getConfiguration();
Path outputFile = new Path(output, "part-r-00000");
List<Vector> clusteredItems = Lists.newArrayList();
String prevClusterId = "";
@@ -147,7 +148,7 @@ public final class TestMinHashClustering
@Test
public void testLinearMinHashMRJob() throws Exception {
String[] args = makeArguments(2, 3, 20, 3, HashType.LINEAR.toString());
- int ret = ToolRunner.run(new Configuration(), new MinHashDriver(), args);
+ int ret = ToolRunner.run(getConfiguration(), new MinHashDriver(), args);
assertEquals("Minhash MR Job failed for " + HashType.LINEAR, 0, ret);
verify(output, 0.2, "Hash Type: LINEAR");
}
@@ -155,7 +156,7 @@ public final class TestMinHashClustering
@Test
public void testPolynomialMinHashMRJob() throws Exception {
String[] args = makeArguments(2, 3, 20, 3, HashType.POLYNOMIAL.toString());
- int ret = ToolRunner.run(new Configuration(), new MinHashDriver(), args);
+ int ret = ToolRunner.run(getConfiguration(), new MinHashDriver(), args);
assertEquals("Minhash MR Job failed for " + HashType.POLYNOMIAL, 0, ret);
verify(output, 0.27, "Hash Type: POLYNOMIAL");
}
@@ -163,7 +164,7 @@ public final class TestMinHashClustering
@Test
public void testMurmurMinHashMRJob() throws Exception {
String[] args = makeArguments(2, 3, 20, 4, HashType.MURMUR.toString());
- int ret = ToolRunner.run(new Configuration(), new MinHashDriver(), args);
+ int ret = ToolRunner.run(getConfiguration(), new MinHashDriver(), args);
assertEquals("Minhash MR Job failed for " + HashType.MURMUR, 0, ret);
verify(output, 0.2, "Hash Type: MURMUR");
}
@@ -171,7 +172,7 @@ public final class TestMinHashClustering
@Test
public void testMurmur3MinHashMRJob() throws Exception {
String[] args = makeArguments(2, 3, 20, 4, HashType.MURMUR3.toString());
- int ret = ToolRunner.run(new Configuration(), new MinHashDriver(), args);
+ int ret = ToolRunner.run(getConfiguration(), new MinHashDriver(), args);
assertEquals("Minhash MR Job failed for " + HashType.MURMUR3, 0, ret);
verify(output, 0.2, "Hash Type: MURMUR");
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java Sat Jun 1 22:24:03 2013
@@ -63,6 +63,12 @@ public abstract class MahoutTestCase ext
}
super.tearDown();
}
+
+ protected final Configuration getConfiguration() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set("hadoop.tmp.dir", getTestTempDir("hadoop" + Math.random()).getAbsolutePath());
+ return conf;
+ }
protected final Path getTestTempDirPath() throws IOException {
if (testTempDirPath == null) {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/FPGrowthTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/FPGrowthTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/FPGrowthTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/FPGrowthTest.java Sat Jun 1 22:24:03 2013
@@ -57,7 +57,7 @@ public final class FPGrowthTest extends
transactions.add(new Pair<List<String>,Long>(Arrays.asList("B", "C"), 1L));
Path path = getTestTempFilePath("fpgrowthTest.dat");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(path.toUri(), conf);
SequenceFile.Writer writer =
@@ -100,7 +100,7 @@ public final class FPGrowthTest extends
transactions.add(new Pair<List<String>,Long>(Arrays.asList("X", "Y"), 10L));
Path path = getTestTempFilePath("fpgrowthTestData1.dat");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(path.toUri(), conf);
System.out.println(fp.generateFList(transactions.iterator(), 2));
SequenceFile.Writer writer =
@@ -138,7 +138,7 @@ public final class FPGrowthTest extends
transactions.add(new Pair<List<String>,Long>(Arrays.asList("X", "Y", "Z"), 11L));
Path path = getTestTempFilePath("fpgrowthTestData2.dat");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
FileSystem fs = FileSystem.get(path.toUri(), conf);
System.out.println(fp.generateFList(transactions.iterator(), 2));
SequenceFile.Writer writer =
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthRetailDataTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthRetailDataTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthRetailDataTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthRetailDataTest.java Sat Jun 1 22:24:03 2013
@@ -62,6 +62,7 @@ public class PFPGrowthRetailDataTest ext
File input = new File(inputDir, "test.txt");
params.set(PFPGrowth.INPUT, input.getAbsolutePath());
params.set(PFPGrowth.OUTPUT, outputDir.getAbsolutePath());
+
Writer writer = Files.newWriter(input, Charsets.UTF_8);
try {
StringRecordIterator it = new StringRecordIterator(new FileLineIterable(Resources.getResource(
@@ -104,7 +105,7 @@ public class PFPGrowthRetailDataTest ext
expectedResults.put(Sets.newHashSet(items), support);
}
- PFPGrowth.runPFPGrowth(params);
+ PFPGrowth.runPFPGrowth(params, getConfiguration());
List<Pair<String,TopKStringPatterns>> frequentPatterns = PFPGrowth.readFrequentPattern(params);
@@ -156,7 +157,7 @@ public class PFPGrowthRetailDataTest ext
Long support = Long.parseLong(supportString.substring(1, supportString.length() - 1));
expectedResults.put(Sets.newHashSet(items), support);
}
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
log.info("Starting Parallel Counting Test: {}", params.get(PFPGrowth.MAX_HEAPSIZE));
PFPGrowth.startParallelCounting(params, conf);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowthTest.java Sat Jun 1 22:24:03 2013
@@ -85,7 +85,7 @@ public final class PFPGrowthTest extends
*/
@Test
public void testStartParallelFPGrowth() throws Exception {
- PFPGrowth.runPFPGrowth(params);
+ PFPGrowth.runPFPGrowth(params, getConfiguration());
List<Pair<String,TopKStringPatterns>> frequentPatterns = PFPGrowth.readFrequentPattern(params);
@@ -102,7 +102,7 @@ public final class PFPGrowthTest extends
*/
@Test
public void testStartParallelFPGrowthInSteps() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
log.info("Starting Parallel Counting Test: {}", params.get(PFPGrowth.MAX_HEAPSIZE));
PFPGrowth.startParallelCounting(params, conf);
log.info("Reading fList Test: {}", params.get(PFPGrowth.MAX_HEAPSIZE));
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java Sat Jun 1 22:24:03 2013
@@ -75,13 +75,15 @@ public final class TestDistributedRowMat
@Test
public void testTranspose() throws Exception {
DistributedRowMatrix m = randomDistributedMatrix(10, 9, 5, 4, 1.0, false);
+ m.setConf(getConfiguration());
DistributedRowMatrix mt = m.transpose();
+ mt.setConf(getConfiguration());
Path tmpPath = getTestTempDirPath();
m.setOutputTempPathString(tmpPath.toString());
Path tmpOutPath = new Path(tmpPath, "/tmpOutTranspose");
mt.setOutputTempPathString(tmpOutPath.toString());
- HadoopUtil.delete(new Configuration(), tmpOutPath);
+ HadoopUtil.delete(getConfiguration(), tmpOutPath);
DistributedRowMatrix mtt = mt.transpose();
assertEquals(m, mtt, EPSILON);
}
@@ -92,6 +94,7 @@ public final class TestDistributedRowMat
SolverTest.randomSequentialAccessSparseMatrix(100, 90, 50, 20, 1.0);
DistributedRowMatrix dm =
randomDistributedMatrix(100, 90, 50, 20, 1.0, false);
+ dm.setConf(getConfiguration());
Vector expected = new DenseVector(50);
for (int i = 0; i < m.numRows(); i++) {
@@ -108,6 +111,7 @@ public final class TestDistributedRowMat
SolverTest.randomSequentialAccessSparseMatrix(100, 90, 0, 0, 1.0);
DistributedRowMatrix dm =
randomDistributedMatrix(100, 90, 0, 0, 1.0, false);
+ dm.setConf(getConfiguration());
Vector expected = new DenseVector(0);
for (int i = 0; i < m.numRows(); i++) {
@@ -124,6 +128,7 @@ public final class TestDistributedRowMat
v.assign(1.0);
Matrix m = SolverTest.randomSequentialAccessSparseMatrix(100, 90, 50, 20, 1.0);
DistributedRowMatrix dm = randomDistributedMatrix(100, 90, 50, 20, 1.0, false);
+ dm.setConf(getConfiguration());
Vector expected = m.times(v);
Vector actual = dm.times(v);
@@ -136,6 +141,7 @@ public final class TestDistributedRowMat
v.assign(1.0);
Matrix m = SolverTest.randomSequentialAccessSparseMatrix(100, 90, 50, 20, 1.0);
DistributedRowMatrix dm = randomDistributedMatrix(100, 90, 50, 20, 1.0, false);
+ dm.setConf(getConfiguration());
Vector expected = m.timesSquared(v);
Vector actual = dm.timesSquared(v);
@@ -149,7 +155,9 @@ public final class TestDistributedRowMat
Matrix expected = inputA.transpose().times(inputB);
DistributedRowMatrix distA = randomDistributedMatrix(20, 19, 15, 5, 10.0, false, "distA");
+ distA.setConf(getConfiguration());
DistributedRowMatrix distB = randomDistributedMatrix(20, 13, 25, 10, 5.0, false, "distB");
+ distB.setConf(getConfiguration());
DistributedRowMatrix product = distA.times(distB);
assertEquals(expected, product, EPSILON);
@@ -247,10 +255,11 @@ public final class TestDistributedRowMat
@Test
public void testTimesVectorTempDirDeletion() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Vector v = new RandomAccessSparseVector(50);
v.assign(1.0);
DistributedRowMatrix dm = randomDistributedMatrix(100, 90, 50, 20, 1.0, false);
+ dm.setConf(conf);
Path outputPath = dm.getOutputTempPath();
FileSystem fs = outputPath.getFileSystem(conf);
@@ -284,10 +293,11 @@ public final class TestDistributedRowMat
@Test
public void testTimesSquaredVectorTempDirDeletion() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Vector v = new RandomAccessSparseVector(50);
v.assign(1.0);
DistributedRowMatrix dm = randomDistributedMatrix(100, 90, 50, 20, 1.0, false);
+ dm.setConf(getConfiguration());
Path outputPath = dm.getOutputTempPath();
FileSystem fs = outputPath.getFileSystem(conf);
@@ -319,8 +329,8 @@ public final class TestDistributedRowMat
assertEquals(0.0, result1.getDistanceSquared(result2), EPSILON);
}
- public static Configuration createInitialConf() {
- Configuration initialConf = new Configuration();
+ public Configuration createInitialConf() throws IOException {
+ Configuration initialConf = getConfiguration();
initialConf.set(TEST_PROPERTY_KEY, TEST_PROPERTY_VALUE);
return initialConf;
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolver.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolver.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolver.java Sat Jun 1 22:24:03 2013
@@ -67,7 +67,7 @@ public final class TestDistributedLanczo
int desiredRank, boolean hdfsBackedState)
throws IOException {
DistributedRowMatrix corpus = getCorpus(symmetric);
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
corpus.setConf(conf);
DistributedLanczosSolver solver = new DistributedLanczosSolver();
Vector intitialVector = DistributedLanczosSolver.getInitialVector(corpus);
@@ -92,7 +92,7 @@ public final class TestDistributedLanczo
public void doTestResumeIteration(boolean symmetric) throws IOException {
DistributedRowMatrix corpus = getCorpus(symmetric);
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
corpus.setConf(conf);
DistributedLanczosSolver solver = new DistributedLanczosSolver();
int rank = 10;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolverCLI.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolverCLI.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolverCLI.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/decomposer/TestDistributedLanczosSolverCLI.java Sat Jun 1 22:24:03 2013
@@ -20,6 +20,7 @@ package org.apache.mahout.math.hadoop.de
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.iterator.sequencefile.SequenceFileValueIterable;
import org.apache.mahout.math.DenseMatrix;
@@ -45,7 +46,7 @@ public final class TestDistributedLanczo
DistributedRowMatrix corpus =
new TestDistributedRowMatrix().randomDenseHierarchicalDistributedMatrix(10, 9, false,
testData.toString());
- corpus.setConf(new Configuration());
+ corpus.setConf(getConfiguration());
Path output = getTestTempDirPath("output");
Path tmp = getTestTempDirPath("tmp");
Path workingDir = getTestTempDirPath("working");
@@ -59,7 +60,7 @@ public final class TestDistributedLanczo
"--symmetric", "false",
"--workingDir", workingDir.toString()
};
- new DistributedLanczosSolver().new DistributedLanczosSolverJob().run(args);
+ ToolRunner.run(getConfiguration(), new DistributedLanczosSolver().new DistributedLanczosSolverJob(), args);
output = getTestTempDirPath("output2");
tmp = getTestTempDirPath("tmp2");
@@ -73,11 +74,11 @@ public final class TestDistributedLanczo
"--symmetric", "false",
"--workingDir", workingDir.toString()
};
- new DistributedLanczosSolver().new DistributedLanczosSolverJob().run(args);
+ ToolRunner.run(getConfiguration(), new DistributedLanczosSolver().new DistributedLanczosSolverJob(), args);
Path rawEigenvectors = new Path(output, DistributedLanczosSolver.RAW_EIGENVECTORS);
Matrix eigenVectors = new DenseMatrix(7, corpus.numCols());
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
int i = 0;
for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(rawEigenvectors, conf)) {
@@ -93,7 +94,7 @@ public final class TestDistributedLanczo
Path testData = getTestTempDirPath("testdata");
DistributedRowMatrix corpus = new TestDistributedRowMatrix()
.randomDenseHierarchicalDistributedMatrix(10, 9, false, testData.toString());
- corpus.setConf(new Configuration());
+ corpus.setConf(getConfiguration());
Path output = getTestTempDirPath("output");
Path tmp = getTestTempDirPath("tmp");
String[] args = {
@@ -106,7 +107,7 @@ public final class TestDistributedLanczo
"--symmetric", "false",
"--cleansvd", "true"
};
- new DistributedLanczosSolver().new DistributedLanczosSolverJob().run(args);
+ ToolRunner.run(getConfiguration(), new DistributedLanczosSolver().new DistributedLanczosSolverJob(), args);
Path cleanEigenvectors = new Path(output, EigenVerificationJob.CLEAN_EIGENVECTORS);
Matrix eigenVectors = new DenseMatrix(6, corpus.numCols());
@@ -124,10 +125,10 @@ public final class TestDistributedLanczo
"--symmetric", "false",
"--cleansvd", "true"
};
- new DistributedLanczosSolver().new DistributedLanczosSolverJob().run(args);
+ ToolRunner.run(getConfiguration(), new DistributedLanczosSolver().new DistributedLanczosSolverJob(), args);
Path cleanEigenvectors2 = new Path(output, EigenVerificationJob.CLEAN_EIGENVECTORS);
Matrix eigenVectors2 = new DenseMatrix(7, corpus.numCols());
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Collection<Double> newEigenValues = Lists.newArrayList();
int i = 0;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestVectorDistanceSimilarityJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestVectorDistanceSimilarityJob.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestVectorDistanceSimilarityJob.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestVectorDistanceSimilarityJob.java Sat Jun 1 22:24:03 2013
@@ -62,7 +62,7 @@ public class TestVectorDistanceSimilarit
@Before
public void setUp() throws Exception {
super.setUp();
- fs = FileSystem.get(new Configuration());
+ fs = FileSystem.get(getConfiguration());
}
@Test
@@ -143,7 +143,7 @@ public class TestVectorDistanceSimilarit
List<VectorWritable> points = getPointsWritable(REFERENCE);
List<VectorWritable> seeds = getPointsWritable(SEEDS);
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf);
@@ -152,7 +152,7 @@ public class TestVectorDistanceSimilarit
output.toString(), optKey(DefaultOptionCreator.DISTANCE_MEASURE_OPTION),
EuclideanDistanceMeasure.class.getName() };
- ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
+ ToolRunner.run(getConfiguration(), new VectorDistanceSimilarityJob(), args);
int expectedOutputSize = SEEDS.length * REFERENCE.length;
int outputSize = Iterables.size(new SequenceFileIterable<StringTuple, DoubleWritable>(new Path(output,
@@ -182,7 +182,7 @@ public class TestVectorDistanceSimilarit
EuclideanDistanceMeasure.class.getName(),
optKey(VectorDistanceSimilarityJob.MAX_DISTANCE), String.valueOf(maxDistance) };
- ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
+ ToolRunner.run(getConfiguration(), new VectorDistanceSimilarityJob(), args);
int outputSize = 0;
@@ -202,7 +202,7 @@ public class TestVectorDistanceSimilarit
Path seedsPath = getTestTempDirPath("seeds");
List<VectorWritable> points = getPointsWritable(REFERENCE);
List<VectorWritable> seeds = getPointsWritable(SEEDS);
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
ClusteringTestUtils.writePointsToFile(points, true, new Path(input, "file1"), fs, conf);
ClusteringTestUtils.writePointsToFile(seeds, true, new Path(seedsPath, "part-seeds"), fs, conf);
String[] args = {optKey(DefaultOptionCreator.INPUT_OPTION), input.toString(),
@@ -211,7 +211,7 @@ public class TestVectorDistanceSimilarit
EuclideanDistanceMeasure.class.getName(),
optKey(VectorDistanceSimilarityJob.OUT_TYPE_KEY), "v"
};
- ToolRunner.run(new Configuration(), new VectorDistanceSimilarityJob(), args);
+ ToolRunner.run(getConfiguration(), new VectorDistanceSimilarityJob(), args);
DummyOutputCollector<Text, VectorWritable> collector = new DummyOutputCollector<Text, VectorWritable>();
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/cooccurrence/RowSimilarityJobTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/cooccurrence/RowSimilarityJobTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/cooccurrence/RowSimilarityJobTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/cooccurrence/RowSimilarityJobTest.java Sat Jun 1 22:24:03 2013
@@ -57,7 +57,7 @@ public class RowSimilarityJobTest extend
outputDir.delete();
File tmpDir = getTestTempDir("tmp");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Path inputPath = new Path(inputFile.getAbsolutePath());
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -98,7 +98,7 @@ public class RowSimilarityJobTest extend
outputDir.delete();
File tmpDir = getTestTempDir("tmp");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Path inputPath = new Path(inputFile.getAbsolutePath());
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -143,7 +143,7 @@ public class RowSimilarityJobTest extend
outputDir.delete();
File tmpDir = getTestTempDir("tmp");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Path inputPath = new Path(inputFile.getAbsolutePath());
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -184,7 +184,7 @@ public class RowSimilarityJobTest extend
File inputFile = getTestTempFile("rows");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Path inputPath = new Path(inputFile.getAbsolutePath());
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolver.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolver.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolver.java Sat Jun 1 22:24:03 2013
@@ -21,9 +21,9 @@ import java.io.File;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
+import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.math.DenseVector;
-import org.apache.mahout.math.MahoutTestCase;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.hadoop.DistributedRowMatrix;
import org.apache.mahout.math.hadoop.TestDistributedRowMatrix;
@@ -47,7 +47,7 @@ public final class TestDistributedConjug
File testData = getTestTempDir("testdata");
DistributedRowMatrix matrix = new TestDistributedRowMatrix().randomDistributedMatrix(
10, 10, 10, 10, 10.0, true, testData.getAbsolutePath());
- matrix.setConf(new Configuration());
+ matrix.setConf(getConfiguration());
Vector vector = randomVector(matrix.numCols(), 10.0);
DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver();
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolverCLI.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolverCLI.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolverCLI.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/solver/TestDistributedConjugateGradientSolverCLI.java Sat Jun 1 22:24:03 2013
@@ -26,6 +26,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.math.DenseVector;
@@ -76,7 +77,7 @@ public final class TestDistributedConjug
@Test
public void testSolver() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
Path testData = getTestTempDirPath("testdata");
DistributedRowMatrix matrix = new TestDistributedRowMatrix().randomDistributedMatrix(
10, 10, 10, 10, 10.0, true, testData.toString());
@@ -99,7 +100,7 @@ public final class TestDistributedConjug
};
DistributedConjugateGradientSolver solver = new DistributedConjugateGradientSolver();
- solver.job().run(args);
+ ToolRunner.run(getConfiguration(), solver.job(), args);
Vector x = loadVector(conf, output);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/stats/BasicStatsTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/stats/BasicStatsTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/stats/BasicStatsTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/stats/BasicStatsTest.java Sat Jun 1 22:24:03 2013
@@ -39,7 +39,7 @@ public final class BasicStatsTest extend
@Before
public void setUp() throws Exception {
super.setUp();
- conf = new Configuration();
+ conf = getConfiguration();
}
@Test
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/ConditionalEntropyTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/ConditionalEntropyTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/ConditionalEntropyTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/ConditionalEntropyTest.java Sat Jun 1 22:24:03 2013
@@ -39,7 +39,7 @@ public final class ConditionalEntropyTes
@Test
public void testConditionalEntropy() throws Exception {
- Configuration configuration = new Configuration();
+ Configuration configuration = getConfiguration();
Path input = getTestTempFilePath("input");
Path output = getTestTempFilePath("output");
FileSystem fileSystem = FileSystem.get(input.toUri(), configuration);
@@ -60,7 +60,7 @@ public final class ConditionalEntropyTes
Tool job = new ConditionalEntropy();
String[] args = { "-i", input.toString(), "-o", output.toString(),
"--tempDir", getTestTempDirPath("tmp").toString() };
- ToolRunner.run(job, args);
+ ToolRunner.run(configuration, job, args);
// check the output
Iterator<DoubleWritable> iteratorNodes =
@@ -69,7 +69,7 @@ public final class ConditionalEntropyTes
PathFilters.logsCRCFilter(),
null,
false,
- new Configuration());
+ getConfiguration());
while (iteratorNodes.hasNext()) {
assertEquals(0.5, iteratorNodes.next().get(), EPSILON);
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/EntropyTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/EntropyTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/EntropyTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/EntropyTest.java Sat Jun 1 22:24:03 2013
@@ -50,7 +50,7 @@ public final class EntropyTest extends M
private void calculateEntropy(String[] content, double expected, String source) throws Exception {
- Configuration configuration = new Configuration();
+ Configuration configuration = getConfiguration();
Path input = getTestTempFilePath("input");
Path output = getTestTempFilePath("output");
FileSystem fileSystem = FileSystem.get(input.toUri(), configuration);
@@ -75,7 +75,7 @@ public final class EntropyTest extends M
String[] args = { "-i", input.toString(), "-o", output.toString(), "-s", source,
"--tempDir", getTestTempDirPath("tmp").toString() };
Entropy job = new Entropy();
- ToolRunner.run(job, args);
+ ToolRunner.run(configuration, job, args);
assertEquals(content.length, job.getNumberItems());
@@ -86,7 +86,7 @@ public final class EntropyTest extends M
PathFilters.logsCRCFilter(),
null,
false,
- new Configuration());
+ getConfiguration());
assertTrue(iteratorNodes.hasNext());
assertEquals(expected, iteratorNodes.next().get(), EPSILON);
}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainRatioTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainRatioTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainRatioTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainRatioTest.java Sat Jun 1 22:24:03 2013
@@ -32,7 +32,7 @@ public final class InformationGainRatioT
@Test
public void testInformationGain() throws Exception {
- Configuration configuration = new Configuration();
+ Configuration configuration = getConfiguration();
Path input = getTestTempFilePath("input");
FileSystem fileSystem = FileSystem.get(input.toUri(), configuration);
@@ -51,7 +51,7 @@ public final class InformationGainRatioT
// run the job
InformationGainRatio job = new InformationGainRatio();
String[] args = { "-i", input.toString(), "--tempDir", getTestTempDirPath("tmp").toString() };
- ToolRunner.run(job, args);
+ ToolRunner.run(configuration, job, args);
// check the output
assertEquals(1.0, job.getEntropy(), EPSILON);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/stats/entropy/InformationGainTest.java Sat Jun 1 22:24:03 2013
@@ -32,7 +32,7 @@ public final class InformationGainTest e
@Test
public void testInformationGain() throws Exception {
- Configuration configuration = new Configuration();
+ Configuration configuration = getConfiguration();
Path input = getTestTempFilePath("input");
FileSystem fileSystem = FileSystem.get(input.toUri(), configuration);
@@ -51,7 +51,7 @@ public final class InformationGainTest e
// run the job
InformationGain job = new InformationGain();
String[] args = { "-i", input.toString(), "--tempDir", getTestTempDirPath("tmp").toString() };
- ToolRunner.run(job, args);
+ ToolRunner.run(configuration, job, args);
// check the output
assertEquals(1.0, job.getEntropy(), EPSILON);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DictionaryVectorizerTest.java Sat Jun 1 22:24:03 2013
@@ -55,7 +55,7 @@ public final class DictionaryVectorizerT
@Before
public void setUp() throws Exception {
super.setUp();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
inputPath = getTestTempFilePath("documents/docs.file");
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -104,7 +104,7 @@ public final class DictionaryVectorizerT
Path tfidf = getTestTempDirPath("output/tfidf");
Path tfidfVectors = new Path(tfidf, "tfidf-vectors");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
DocumentProcessor.tokenizeDocuments(inputPath, analyzer, tokenizedDocuments, conf);
DictionaryVectorizer.createTermFrequencyVectors(tokenizedDocuments,
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/DocumentProcessorTest.java Sat Jun 1 22:24:03 2013
@@ -42,7 +42,7 @@ public class DocumentProcessorTest exten
@Test
public void testTokenizeDocuments() throws Exception {
- Configuration configuration = new Configuration();
+ Configuration configuration = getConfiguration();
Path input = new Path(getTestTempDirPath(), "inputDir");
Path output = new Path(getTestTempDirPath(), "outputDir");
FileSystem fs = FileSystem.get(input.toUri(), configuration);
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/EncodedVectorsFromSequenceFilesTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/EncodedVectorsFromSequenceFilesTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/EncodedVectorsFromSequenceFilesTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/EncodedVectorsFromSequenceFilesTest.java Sat Jun 1 22:24:03 2013
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.Pair;
import org.apache.mahout.common.iterator.sequencefile.PathFilters;
@@ -48,7 +49,7 @@ public class EncodedVectorsFromSequenceF
@Before
public void setUp() throws Exception {
super.setUp();
- conf = new Configuration();
+ conf = getConfiguration();
inputPath = getTestTempFilePath("documents/docs.file");
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -107,7 +108,7 @@ public class EncodedVectorsFromSequenceF
String[] args = argList.toArray(new String[argList.size()]);
- EncodedVectorsFromSequenceFiles.main(args);
+ ToolRunner.run(getConfiguration(), new EncodedVectorsFromSequenceFiles(), args);
SequenceFileDirIterator<Text, VectorWritable> iter = new SequenceFileDirIterator<Text, VectorWritable>(outputPath, PathType.LIST, PathFilters.partFilter(), null, true, conf);
int seen = 0;
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/HighDFWordsPrunerTest.java Sat Jun 1 22:24:03 2013
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.Pair;
import org.apache.mahout.common.iterator.sequencefile.PathFilters;
@@ -50,7 +51,7 @@ public class HighDFWordsPrunerTest exten
@Before
public void setUp() throws Exception {
super.setUp();
- conf = new Configuration();
+ conf = getConfiguration();
inputPath = getTestTempFilePath("documents/docs.file");
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -92,6 +93,8 @@ public class HighDFWordsPrunerTest exten
argList.add(inputPath.toString());
argList.add("-o");
argList.add(outputPath.toString());
+ argList.add("--mapred");
+ argList.add(getTestTempDir("mapred" + Math.random()).getAbsolutePath());
if (prune) {
argList.add("-xs");
argList.add("3"); // we prune all words that are outside 3*sigma
@@ -105,7 +108,7 @@ public class HighDFWordsPrunerTest exten
String[] args = argList.toArray(new String[argList.size()]);
- SparseVectorsFromSequenceFiles.main(args);
+ ToolRunner.run(getConfiguration(), new SparseVectorsFromSequenceFiles(), args);
Path dictionary = new Path(outputPath, "dictionary.file-0");
Path tfVectors = new Path(outputPath, "tf-vectors");
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFilesTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFilesTest.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFilesTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFilesTest.java Sat Jun 1 22:24:03 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.common.MahoutTestCase;
import org.apache.mahout.common.iterator.sequencefile.PathFilters;
import org.apache.mahout.common.iterator.sequencefile.PathType;
@@ -43,7 +44,7 @@ public class SparseVectorsFromSequenceFi
private Path inputPath;
private void setupDocs() throws IOException {
- conf = new Configuration();
+ conf = getConfiguration();
inputPath = getTestTempFilePath("documents/docs.file");
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -88,8 +89,7 @@ public class SparseVectorsFromSequenceFi
@Test
public void testPruning() throws Exception {
- conf = new Configuration();
-
+ conf = getConfiguration();
inputPath = getTestTempFilePath("documents/docs.file");
FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
@@ -126,7 +126,7 @@ public class SparseVectorsFromSequenceFi
@Test
public void testPruningTF() throws Exception {
- conf = new Configuration();
+ conf = getConfiguration();
FileSystem fs = FileSystem.get(conf);
inputPath = getTestTempFilePath("documents/docs.file");
@@ -164,7 +164,6 @@ public class SparseVectorsFromSequenceFi
private Path runTest(boolean tfWeighting, boolean sequential, boolean named, double maxDFSigma, int numDocs) throws Exception {
Path outputPath = getTestTempFilePath("output");
-
List<String> argList = new LinkedList<String>();
argList.add("-i");
argList.add(inputPath.toString());
@@ -188,7 +187,7 @@ public class SparseVectorsFromSequenceFi
}
String[] args = argList.toArray(new String[argList.size()]);
- SparseVectorsFromSequenceFiles.main(args);
+ ToolRunner.run(getConfiguration(), new SparseVectorsFromSequenceFiles(), args);
Path tfVectors = new Path(outputPath, "tf-vectors");
Path tfidfVectors = new Path(outputPath, "tfidf-vectors");
Modified: mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SplitInput.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SplitInput.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SplitInput.java (original)
+++ mahout/trunk/integration/src/main/java/org/apache/mahout/utils/SplitInput.java Sat Jun 1 22:24:03 2013
@@ -280,6 +280,13 @@ public class SplitInput extends Abstract
if (conf == null) {
conf = new Configuration();
}
+ splitDirectory(conf, inputDir);
+ }
+
+ /*
+ * See also splitDirectory(Path inputDir)
+ * */
+ public void splitDirectory(Configuration conf, Path inputDir) throws IOException, ClassNotFoundException, InterruptedException {
FileSystem fs = inputDir.getFileSystem(conf);
if (fs.getFileStatus(inputDir) == null) {
throw new IOException(inputDir + " does not exist");
@@ -289,8 +296,8 @@ public class SplitInput extends Abstract
}
if (useMapRed) {
- SplitInputJob.run(new Configuration(), inputDir, mapRedOutputDirectory,
- keepPct, testRandomSelectionPct);
+ SplitInputJob.run(conf, inputDir, mapRedOutputDirectory,
+ keepPct, testRandomSelectionPct);
} else {
// input dir contains one file per category.
FileStatus[] fileStats = fs.listStatus(inputDir, PathFilters.logsCRCFilter());
@@ -302,7 +309,6 @@ public class SplitInput extends Abstract
}
}
-
/**
* Perform a split on the specified input file. Results will be written to files of the same name in the specified
* training and test output directories. The {@link #validate()} method is called prior to executing the split.
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java Sat Jun 1 22:24:03 2013
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.ToolRunner;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -198,7 +199,7 @@ public final class TestClusterDumper ext
DistanceMeasure measure = new EuclideanDistanceMeasure();
// now run the Canopy job to prime kMeans canopies
Path output = getTestTempDirPath("output");
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
CanopyDriver.run(conf, getTestTempDirPath("testdata"), output, measure, 8,
4, false, 0.0, true);
// now run the KMeans job
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java?rev=1488627&r1=1488626&r2=1488627&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/clustering/TestClusterEvaluator.java Sat Jun 1 22:24:03 2013
@@ -150,7 +150,7 @@ public final class TestClusterEvaluator
public void testRepresentativePoints() throws Exception {
ClusteringTestUtils.writePointsToFile(referenceData, new Path(testdata, "file1"), fs, conf);
DistanceMeasure measure = new EuclideanDistanceMeasure();
- Configuration conf = new Configuration();
+ Configuration conf = getConfiguration();
// run using MR reference point calculation
CanopyDriver.run(conf, testdata, output, measure, 3.1, 1.1, true, 0.0, true);
int numIterations = 2;