You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sm...@apache.org on 2014/04/15 04:35:15 UTC
svn commit: r1587387 - in /mahout/trunk: CHANGELOG
core/src/main/java/org/apache/mahout/classifier/naivebayes/test/TestNaiveBayesDriver.java
Author: smarthi
Date: Tue Apr 15 02:35:14 2014
New Revision: 1587387
URL: http://svn.apache.org/r1587387
Log:
MAHOUT-1503: TestNaiveBayesDriver fails in sequential mode
Modified:
mahout/trunk/CHANGELOG
mahout/trunk/core/src/main/java/org/apache/mahout/classifier/naivebayes/test/TestNaiveBayesDriver.java
Modified: mahout/trunk/CHANGELOG
URL: http://svn.apache.org/viewvc/mahout/trunk/CHANGELOG?rev=1587387&r1=1587386&r2=1587387&view=diff
==============================================================================
--- mahout/trunk/CHANGELOG (original)
+++ mahout/trunk/CHANGELOG Tue Apr 15 02:35:14 2014
@@ -6,6 +6,8 @@ Release 1.0 - unreleased
MAHOUT-1508: Performance problems with sparse matrices (ssc)
+ MAHOUT-1503: TestNaiveBayesDriver fails in sequential mode (Andrew Palumbo, smarthi)
+
MAHOUT-1501: ClusterOutputPostProcessorDriver has private default constructor (ssc)
MAHOUT-1491: Spectral KMeans Clustering doesn't clean its /tmp dir and fails when seeing it again (smarthi)
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/classifier/naivebayes/test/TestNaiveBayesDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/naivebayes/test/TestNaiveBayesDriver.java?rev=1587387&r1=1587386&r2=1587387&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/classifier/naivebayes/test/TestNaiveBayesDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/classifier/naivebayes/test/TestNaiveBayesDriver.java Tue Apr 15 02:35:14 2014
@@ -22,11 +22,11 @@ import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
+import com.google.common.io.Closeables;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
@@ -82,61 +82,66 @@ public class TestNaiveBayesDriver extend
if (hasOption(DefaultOptionCreator.OVERWRITE_OPTION)) {
HadoopUtil.delete(getConf(), getOutputPath());
}
-
- boolean complementary = hasOption("testComplementary");
+
boolean sequential = hasOption("runSequential");
+ boolean succeeded;
if (sequential) {
- FileSystem fs = FileSystem.get(getConf());
- NaiveBayesModel model = NaiveBayesModel.materialize(new Path(getOption("model")), getConf());
- AbstractNaiveBayesClassifier classifier;
- if (complementary) {
- classifier = new ComplementaryNaiveBayesClassifier(model);
- } else {
- classifier = new StandardNaiveBayesClassifier(model);
- }
- SequenceFile.Writer writer =
- new SequenceFile.Writer(fs, getConf(), getOutputPath(), Text.class, VectorWritable.class);
- Reader reader = new Reader(fs, getInputPath(), getConf());
- Text key = new Text();
- VectorWritable vw = new VectorWritable();
- while (reader.next(key, vw)) {
- writer.append(new Text(SLASH.split(key.toString())[1]),
- new VectorWritable(classifier.classifyFull(vw.get())));
- }
- writer.close();
- reader.close();
+ runSequential();
} else {
- boolean succeeded = runMapReduce(parsedArgs);
+ succeeded = runMapReduce();
if (!succeeded) {
return -1;
}
}
-
+
//load the labels
Map<Integer, String> labelMap = BayesUtils.readLabelIndex(getConf(), new Path(getOption("labelIndex")));
//loop over the results and create the confusion matrix
SequenceFileDirIterable<Text, VectorWritable> dirIterable =
- new SequenceFileDirIterable<Text, VectorWritable>(getOutputPath(),
- PathType.LIST,
- PathFilters.partFilter(),
- getConf());
+ new SequenceFileDirIterable<Text, VectorWritable>(getOutputPath(), PathType.LIST, PathFilters.partFilter(), getConf());
ResultAnalyzer analyzer = new ResultAnalyzer(labelMap.values(), "DEFAULT");
analyzeResults(labelMap, dirIterable, analyzer);
- log.info("{} Results: {}", complementary ? "Complementary" : "Standard NB", analyzer);
+ log.info("{} Results: {}", hasOption("testComplementary") ? "Complementary" : "Standard NB", analyzer);
return 0;
}
- private boolean runMapReduce(Map<String, List<String>> parsedArgs) throws IOException,
+ private void runSequential() throws IOException {
+ boolean complementary = hasOption("testComplementary");
+ FileSystem fs = FileSystem.get(getConf());
+ NaiveBayesModel model = NaiveBayesModel.materialize(new Path(getOption("model")), getConf());
+ AbstractNaiveBayesClassifier classifier;
+ if (complementary) {
+ classifier = new ComplementaryNaiveBayesClassifier(model);
+ } else {
+ classifier = new StandardNaiveBayesClassifier(model);
+ }
+ SequenceFile.Writer writer =
+ SequenceFile.createWriter(fs, getConf(), new Path(getOutputPath(), "part-r-00000"), Text.class, VectorWritable.class);
+
+ try {
+ SequenceFileDirIterable<Text, VectorWritable> dirIterable =
+ new SequenceFileDirIterable<Text, VectorWritable>(getInputPath(), PathType.LIST, PathFilters.partFilter(), getConf());
+ // loop through the part-r-* files in getInputPath() and get classification scores for all entries
+ for (Pair<Text, VectorWritable> pair : dirIterable) {
+ writer.append(new Text(SLASH.split(pair.getFirst().toString())[1]),
+ new VectorWritable(classifier.classifyFull(pair.getSecond().get())));
+ }
+ } finally {
+ Closeables.close(writer, false);
+ }
+ }
+
+ private boolean runMapReduce() throws IOException,
InterruptedException, ClassNotFoundException {
Path model = new Path(getOption("model"));
HadoopUtil.cacheFiles(model, getConf());
//the output key is the expected value, the output value are the scores for all the labels
Job testJob = prepareJob(getInputPath(), getOutputPath(), SequenceFileInputFormat.class, BayesTestMapper.class,
- Text.class, VectorWritable.class, SequenceFileOutputFormat.class);
+ Text.class, VectorWritable.class, SequenceFileOutputFormat.class);
//testJob.getConfiguration().set(LABEL_KEY, getOption("--labels"));
-
+
//boolean complementary = parsedArgs.containsKey("testComplementary"); //always result to false as key in hash map is "--testComplementary"
boolean complementary = hasOption("testComplementary"); //or complementary = parsedArgs.containsKey("--testComplementary");
testJob.getConfiguration().set(COMPLEMENTARY, String.valueOf(complementary));