You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by ss...@apache.org on 2014/04/28 17:46:53 UTC
svn commit: r1590687 - in /mahout/trunk: ./
examples/src/main/java/org/apache/mahout/classifier/sgd/
integration/src/test/java/org/apache/mahout/text/
math/src/test/java/org/apache/mahout/math/
mrlegacy/src/test/java/org/apache/mahout/clustering/fuzzyk...
Author: ssc
Date: Mon Apr 28 15:46:53 2014
New Revision: 1590687
URL: http://svn.apache.org/r1590687
Log:
MAHOUT-1310 Mahout support windows
Modified:
mahout/trunk/CHANGELOG
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SGDHelper.java
mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java
mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromMailArchivesTest.java
mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java
mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
mahout/trunk/pom.xml
mahout/trunk/spark/pom.xml
Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Mon Apr 28 15:46:53 2014
@@ -2,6 +2,8 @@ Mahout Change Log
Release 1.0 - unreleased
+ MAHOUT-1310: Mahout support windows (Sergey Svinarchuk via ssc)
+
MAHOUT-1520: Fix links in Mahout website documentation (Saleem Ansari via smarthi)
MAHOUT-1523: Remove @author tags in sparkbindings (ssc)
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SGDHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SGDHelper.java?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SGDHelper.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/SGDHelper.java Mon Apr 28 15:46:53 2014
@@ -138,8 +138,8 @@ public final class SGDHelper {
}
if (k % (bump * scale) == 0) {
if (best != null) {
- ModelSerializer.writeBinary("/tmp/news-group-" + k + ".model",
- best.getPayload().getLearner().getModels().get(0));
+ File modelFile = new File(System.getProperty("java.io.tmpdir"), "news-group-" + k + ".model");
+ ModelSerializer.writeBinary(modelFile.getAbsolutePath(), best.getPayload().getLearner().getModels().get(0));
}
info.setStep(info.getStep() + 0.25);
Modified: mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java
URL: http://svn.apache.org/viewvc/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java (original)
+++ mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/sgd/TrainNewsGroups.java Mon Apr 28 15:46:53 2014
@@ -132,8 +132,9 @@ public final class TrainNewsGroups {
SGDHelper.dissect(leakType, newsGroups, learningAlgorithm, files, overallCounts);
System.out.println("exiting main");
- ModelSerializer.writeBinary("/tmp/news-group.model",
- learningAlgorithm.getBest().getPayload().getLearner().getModels().get(0));
+ File modelFile = new File(System.getProperty("java.io.tmpdir"), "news-group.model");
+ ModelSerializer.writeBinary(modelFile.getAbsolutePath(),
+ learningAlgorithm.getBest().getPayload().getLearner().getModels().get(0));
List<Integer> counts = Lists.newArrayList();
System.out.println("Word counts");
Modified: mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromMailArchivesTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromMailArchivesTest.java?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromMailArchivesTest.java (original)
+++ mahout/trunk/integration/src/test/java/org/apache/mahout/text/SequenceFilesFromMailArchivesTest.java Mon Apr 28 15:46:53 2014
@@ -22,6 +22,7 @@ import java.util.zip.GZIPOutputStream;
import com.google.common.io.Closeables;
+import org.apache.commons.lang3.SystemUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -161,23 +162,39 @@ public final class SequenceFilesFromMail
File parentFileSubSubDir = new File(new File(new File(new File("TEST"), "subdir"), "subsubdir"), "mail-messages-2.gz");
- Assert.assertEquals(new File(parentFileSubSubDir, testVars[0][0]).toString(), record.getFirst().toString());
+ String expected = record.getFirst().toString();
+ if (SystemUtils.IS_OS_WINDOWS) {
+ expected = expected.replace("/", "\\");
+ }
+ Assert.assertEquals(new File(parentFileSubSubDir, testVars[0][0]).toString(), expected);
Assert.assertEquals(testVars[0][1] + testVars[0][2], record.getSecond().toString());
Assert.assertTrue("Second key/value pair not found!", iterator.hasNext());
record = iterator.next();
- Assert.assertEquals(new File(parentFileSubSubDir, testVars[1][0]).toString(), record.getFirst().toString());
+ expected = record.getFirst().toString();
+ if (SystemUtils.IS_OS_WINDOWS) {
+ expected = expected.replace("/", "\\");
+ }
+ Assert.assertEquals(new File(parentFileSubSubDir, testVars[1][0]).toString(), expected);
Assert.assertEquals(testVars[1][1] + testVars[1][2], record.getSecond().toString());
// test other file
File parentFile = new File(new File(new File("TEST"), "subdir"), "mail-messages.gz");
record = iterator.next();
- Assert.assertEquals(new File(parentFile, testVars[0][0]).toString(), record.getFirst().toString());
+ expected = record.getFirst().toString();
+ if (SystemUtils.IS_OS_WINDOWS) {
+ expected = expected.replace("/", "\\");
+ }
+ Assert.assertEquals(new File(parentFile, testVars[0][0]).toString(), expected);
Assert.assertEquals(testVars[0][1] + testVars[0][2], record.getSecond().toString());
Assert.assertTrue("Second key/value pair not found!", iterator.hasNext());
record = iterator.next();
- Assert.assertEquals(new File(parentFile, testVars[1][0]).toString(), record.getFirst().toString());
+ expected = record.getFirst().toString();
+ if (SystemUtils.IS_OS_WINDOWS) {
+ expected = expected.replace("/", "\\");
+ }
+ Assert.assertEquals(new File(parentFile, testVars[1][0]).toString(), expected);
Assert.assertEquals(testVars[1][1] + testVars[1][2], record.getSecond().toString());
Assert.assertFalse("Only four key/value pairs expected!", iterator.hasNext());
}
Modified: mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java (original)
+++ mahout/trunk/math/src/test/java/org/apache/mahout/math/MahoutTestCase.java Mon Apr 28 15:46:53 2014
@@ -36,7 +36,7 @@ import com.carrotsearch.randomizedtestin
/**
* Superclass of all Mahout test cases.
*/
-@ThreadLeakScope(ThreadLeakScope.Scope.SUITE)
+@ThreadLeakScope(ThreadLeakScope.Scope.NONE)
@ThreadLeakAction({ThreadLeakAction.Action.WARN, ThreadLeakAction.Action.INTERRUPT})
@ThreadLeakLingering(linger = 20000) // Wait a bit longer for leaked threads to die.
@ThreadLeakZombies(ThreadLeakZombies.Consequence.IGNORE_REMAINING_TESTS)
Modified: mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java (original)
+++ mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java Mon Apr 28 15:46:53 2014
@@ -87,7 +87,7 @@ public final class TestFuzzyKmeansCluste
}
// now run the Job using the run() command line options.
- Path output = getTestTempDirPath("output");
+ Path output = getTestTempDirPath("output" + k);
/* FuzzyKMeansDriver.runJob(pointsPath,
clustersPath,
output,
@@ -160,7 +160,7 @@ public final class TestFuzzyKmeansCluste
}
// now run the Job using the run() command line options.
- Path output = getTestTempDirPath("output");
+ Path output = getTestTempDirPath("output" + k);
/* FuzzyKMeansDriver.runJob(pointsPath,
clustersPath,
output,
Modified: mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
URL: http://svn.apache.org/viewvc/mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java (original)
+++ mahout/trunk/mrlegacy/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java Mon Apr 28 15:46:53 2014
@@ -160,7 +160,7 @@ public final class TestKmeansClustering
Closeables.close(writer, false);
}
// now run the Job
- Path outputPath = getTestTempDirPath("output");
+ Path outputPath = getTestTempDirPath("output" + k);
String[] args = {optKey(DefaultOptionCreator.INPUT_OPTION), pointsPath.toString(),
optKey(DefaultOptionCreator.CLUSTERS_IN_OPTION), clustersPath.toString(),
optKey(DefaultOptionCreator.OUTPUT_OPTION), outputPath.toString(),
@@ -214,7 +214,7 @@ public final class TestKmeansClustering
Closeables.close(writer, false);
}
// now run the Job
- Path outputPath = getTestTempDirPath("output");
+ Path outputPath = getTestTempDirPath("output" + k);
String[] args = {optKey(DefaultOptionCreator.INPUT_OPTION), pointsPath.toString(),
optKey(DefaultOptionCreator.CLUSTERS_IN_OPTION), clustersPath.toString(),
optKey(DefaultOptionCreator.OUTPUT_OPTION), outputPath.toString(),
@@ -269,7 +269,7 @@ public final class TestKmeansClustering
Closeables.close(writer, false);
}
// now run the Job
- Path outputPath = getTestTempDirPath("output");
+ Path outputPath = getTestTempDirPath("output" + k);
String[] args = {optKey(DefaultOptionCreator.INPUT_OPTION), pointsPath.toString(),
optKey(DefaultOptionCreator.CLUSTERS_IN_OPTION), clustersPath.toString(),
optKey(DefaultOptionCreator.OUTPUT_OPTION), outputPath.toString(),
Modified: mahout/trunk/pom.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/pom.xml?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/pom.xml (original)
+++ mahout/trunk/pom.xml Mon Apr 28 15:46:53 2014
@@ -534,7 +534,7 @@
<configuration>
<forkCount>2</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx512m -Djava.security.manager
+ <argLine>-Xmx512m -Djava.security.manager -Djava.library.path=${env.HADOOP_HOME}\bin
-Djava.security.policy=${project.build.directory}/../../buildtools/src/test/resources/java.policy</argLine>
<argLine>-Djava.security.auth.login.config=${project.build.directory}/../../buildtools/src/test/resources/jaas.config</argLine>
<testFailureIgnore>false</testFailureIgnore>
Modified: mahout/trunk/spark/pom.xml
URL: http://svn.apache.org/viewvc/mahout/trunk/spark/pom.xml?rev=1590687&r1=1590686&r2=1590687&view=diff
==============================================================================
--- mahout/trunk/spark/pom.xml (original)
+++ mahout/trunk/spark/pom.xml Mon Apr 28 15:46:53 2014
@@ -212,6 +212,55 @@
</dependency>
</dependencies>
</profile>
+
+ <profile>
+ <id>hadoop2</id>
+ <activation>
+ <property>
+ <name>hadoop2.version</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop2.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.mahout</groupId>
+ <artifactId>mahout-mrlegacy</artifactId>
+ <version>${pom.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>asm</groupId>
+ <artifactId>asm</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
<dependencies>