You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/07/10 11:35:28 UTC
svn commit: r792856 [6/13] - in /lucene/mahout/trunk/core/src:
main/java/org/apache/mahout/cf/taste/common/
main/java/org/apache/mahout/cf/taste/eval/
main/java/org/apache/mahout/cf/taste/hadoop/
main/java/org/apache/mahout/cf/taste/impl/common/ main/j...
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java Fri Jul 10 09:35:19 2009
@@ -22,23 +22,21 @@
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.util.GenericsUtil;
import org.apache.mahout.classifier.bayes.io.SequenceFileModelReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
import java.io.IOException;
+import java.util.Map;
-/**
- * Create and run the Bayes Theta Normalization Step.
- */
+/** Create and run the Bayes Theta Normalization Step. */
public class BayesThetaNormalizerDriver {
private static final Logger log = LoggerFactory.getLogger(BayesThetaNormalizerDriver.class);
@@ -47,12 +45,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the the interim filesas a
- * {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the the interim filesas a {@link
+ * org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -65,13 +61,13 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(BayesThetaNormalizerDriver.class);
-
+
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(DoubleWritable.class);
@@ -82,30 +78,31 @@
//conf.setNumReduceTasks(1);
conf.setMapperClass(BayesThetaNormalizerMapper.class);
conf.setInputFormat(SequenceFileInputFormat.class);
- conf.setCombinerClass(BayesThetaNormalizerReducer.class);
- conf.setReducerClass(BayesThetaNormalizerReducer.class);
+ conf.setCombinerClass(BayesThetaNormalizerReducer.class);
+ conf.setReducerClass(BayesThetaNormalizerReducer.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.set("io.serializations",
- "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
// Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code
-
+
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
- Path Sigma_kFiles = new Path(output+"/trainer-weights/Sigma_k/*");
- Map<String,Double> labelWeightSum = SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf);
- DefaultStringifier<Map<String,Double>> mapStringifier =
- new DefaultStringifier<Map<String,Double>>(conf, GenericsUtil.getClass(labelWeightSum));
+ Path Sigma_kFiles = new Path(output + "/trainer-weights/Sigma_k/*");
+ Map<String, Double> labelWeightSum = SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf);
+ DefaultStringifier<Map<String, Double>> mapStringifier =
+ new DefaultStringifier<Map<String, Double>>(conf, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
log.info("Sigma_k for Each Label");
- Map<String,Double> c = mapStringifier.fromString(labelWeightSumString);
+ Map<String, Double> c = mapStringifier.fromString(labelWeightSumString);
log.info("{}", c);
conf.set("cnaivebayes.sigma_k", labelWeightSumString);
- Path sigma_kSigma_jFile = new Path(output+"/trainer-weights/Sigma_kSigma_j/*");
+ Path sigma_kSigma_jFile = new Path(output + "/trainer-weights/Sigma_kSigma_j/*");
double sigma_jSigma_k = SequenceFileModelReader.readSigma_jSigma_k(dfs, sigma_kSigma_jFile, conf);
DefaultStringifier<Double> stringifier = new DefaultStringifier<Double>(conf, Double.class);
String sigma_jSigma_kString = stringifier.toString(sigma_jSigma_k);
@@ -115,7 +112,7 @@
log.info("{}", retSigma_jSigma_k);
conf.set("cnaivebayes.sigma_jSigma_k", sigma_jSigma_kString);
- Path vocabCountFile = new Path(output+"/trainer-tfIdf/trainer-vocabCount/*");
+ Path vocabCountFile = new Path(output + "/trainer-tfIdf/trainer-vocabCount/*");
double vocabCount = SequenceFileModelReader.readVocabCount(dfs, vocabCountFile, conf);
String vocabCountString = stringifier.toString(vocabCount);
@@ -127,6 +124,6 @@
client.setConf(conf);
JobClient.runJob(conf);
-
+
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java Fri Jul 10 09:35:19 2009
@@ -38,22 +38,19 @@
private static final Logger log = LoggerFactory.getLogger(BayesThetaNormalizerMapper.class);
- private Map<String,Double> labelWeightSum = null;
+ private Map<String, Double> labelWeightSum = null;
private double sigma_jSigma_k = 0.0;
private double vocabCount = 0.0;
/**
* We need to calculate the thetaNormalization factor of each label
- *
- * @param key The label,feature pair
- * @param value The tfIdf of the pair
- * @param output
- * @param reporter
- * @throws IOException
+ *
+ * @param key The label,feature pair
+ * @param value The tfIdf of the pair
*/
@Override
public void map(Text key, DoubleWritable value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
@@ -68,9 +65,9 @@
public void configure(JobConf job) {
try {
if (labelWeightSum == null) {
- labelWeightSum = new HashMap<String,Double>();
+ labelWeightSum = new HashMap<String, Double>();
- DefaultStringifier<Map<String,Double>> mapStringifier = new DefaultStringifier<Map<String,Double>>(
+ DefaultStringifier<Map<String, Double>> mapStringifier = new DefaultStringifier<Map<String, Double>>(
job, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java Fri Jul 10 09:35:19 2009
@@ -27,20 +27,17 @@
import java.io.IOException;
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner beacuse only two values should be there
- * inside the values
- */
+/** Can also be used as a local Combiner beacuse only two values should be there inside the values */
public class BayesThetaNormalizerReducer extends MapReduceBase implements
Reducer<Text, DoubleWritable, Text, DoubleWritable> {
-
+
@Override
public void reduce(Text key, Iterator<DoubleWritable> values,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
// Key is label,word, value is the number of times we've seen this label
// word per local node. Output is the same
-
+
//String token = key.toString();
double weightSumPerLabel = 0.0;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java Fri Jul 10 09:35:19 2009
@@ -22,19 +22,17 @@
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.KeyValueTextInputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-/**
- * Create and run the Bayes Feature Reader Step.
- */
+/** Create and run the Bayes Feature Reader Step. */
public class BayesFeatureDriver {
private static final Logger log = LoggerFactory.getLogger(BayesFeatureDriver.class);
@@ -43,11 +41,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the interim files as a {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the interim files as a {@link
+ * org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -60,8 +57,8 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output, int gramSize) throws IOException {
JobClient client = new JobClient();
@@ -79,16 +76,17 @@
conf.setInputFormat(KeyValueTextInputFormat.class);
conf.setCombinerClass(BayesFeatureReducer.class);
- conf.setReducerClass(BayesFeatureReducer.class);
+ conf.setReducerClass(BayesFeatureReducer.class);
conf.setOutputFormat(BayesFeatureOutputFormat.class);
conf.set("io.serializations",
- "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
// Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
DefaultStringifier<Integer> intStringifier = new DefaultStringifier<Integer>(conf, Integer.class);
String gramSizeString = intStringifier.toString(gramSize);
@@ -98,6 +96,6 @@
client.setConf(conf);
JobClient.runJob(conf);
-
+
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java Fri Jul 10 09:35:19 2009
@@ -32,12 +32,10 @@
import java.io.IOException;
import java.util.HashMap;
-import java.util.Map;
import java.util.List;
+import java.util.Map;
-/**
- * Reads the input train set(preprocessed using the {@link BayesFileFormatter}).
- */
+/** Reads the input train set(preprocessed using the {@link BayesFileFormatter}). */
public class BayesFeatureMapper extends MapReduceBase implements
Mapper<Text, Text, Text, DoubleWritable> {
@@ -50,22 +48,19 @@
private int gramSize = 1;
/**
- * We need to count the number of times we've seen a term with a given label
- * and we need to output that. But this Mapper does more than just outputing the count. It first does weight
- * normalisation.
- * Secondly, it outputs for each unique word in a document value 1 for summing up as the Term Document Frequency.
- * Which later is used to calculate the Idf
- * Thirdly, it outputs for each label the number of times a document was seen(Also used in Idf Calculation)
- *
- * @param key The label
- * @param value the features (all unique) associated w/ this label
- * @param output The OutputCollector to write the results to
+ * We need to count the number of times we've seen a term with a given label and we need to output that. But this
+ * Mapper does more than just outputing the count. It first does weight normalisation. Secondly, it outputs for each
+ * unique word in a document value 1 for summing up as the Term Document Frequency. Which later is used to calculate
+ * the Idf Thirdly, it outputs for each label the number of times a document was seen(Also used in Idf Calculation)
+ *
+ * @param key The label
+ * @param value the features (all unique) associated w/ this label
+ * @param output The OutputCollector to write the results to
* @param reporter Not used
- * @throws IOException
*/
@Override
public void map(Text key, Text value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
//String line = value.toString();
String label = key.toString();
@@ -75,8 +70,8 @@
StringBuilder builder = new StringBuilder(label);
builder.ensureCapacity(32);// make sure we have a reasonably size buffer to
- // begin with
- List<String> ngrams = Model.generateNGramsWithoutLabel(value.toString(), gramSize);
+ // begin with
+ List<String> ngrams = Model.generateNGramsWithoutLabel(value.toString(), gramSize);
for (String ngram : ngrams) {
int[] count = wordList.get(ngram);
if (count == null) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java Fri Jul 10 09:35:19 2009
@@ -30,18 +30,19 @@
import java.io.IOException;
/**
- * This class extends the MultipleOutputFormat, allowing to write the output data to different output files in sequence file output format.
+ * This class extends the MultipleOutputFormat, allowing to write the output data to different output files in sequence
+ * file output format.
*/
-public class BayesFeatureOutputFormat extends MultipleOutputFormat<WritableComparable<?>,Writable>{
+public class BayesFeatureOutputFormat extends MultipleOutputFormat<WritableComparable<?>, Writable> {
- private SequenceFileOutputFormat<WritableComparable<?>,Writable> theSequenceFileOutputFormat = null;
+ private SequenceFileOutputFormat<WritableComparable<?>, Writable> theSequenceFileOutputFormat = null;
@Override
protected RecordWriter<WritableComparable<?>, Writable> getBaseRecordWriter(
FileSystem fs, JobConf job, String name, Progressable arg3)
throws IOException {
if (theSequenceFileOutputFormat == null) {
- theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable<?>,Writable>();
+ theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable<?>, Writable>();
}
return theSequenceFileOutputFormat.getRecordWriter(fs, job, name, arg3);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java Fri Jul 10 09:35:19 2009
@@ -27,9 +27,7 @@
import java.io.IOException;
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner. A simple summing reducer
- */
+/** Can also be used as a local Combiner. A simple summing reducer */
public class BayesFeatureReducer extends MapReduceBase
implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
@@ -39,7 +37,7 @@
OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the number of times we've seen this label word per local node. Output is the same
-
+
double sum = 0.0;
while (values.hasNext()) {
sum += values.next().get();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java Fri Jul 10 09:35:19 2009
@@ -22,22 +22,20 @@
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.util.GenericsUtil;
import org.apache.mahout.classifier.bayes.io.SequenceFileModelReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
import java.io.IOException;
+import java.util.Map;
-/**
- * The Driver which drives the Tf-Idf Generation
- */
+/** The Driver which drives the Tf-Idf Generation */
public class BayesTfIdfDriver {
private static final Logger log = LoggerFactory.getLogger(BayesTfIdfDriver.class);
@@ -46,12 +44,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the interim files as a
- * {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the interim files as a {@link
+ * org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -64,13 +60,13 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(BayesTfIdfDriver.class);
-
+
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(DoubleWritable.class);
@@ -81,30 +77,31 @@
Path outPath = new Path(output + "/trainer-tfIdf");
FileOutputFormat.setOutputPath(conf, outPath);
conf.setNumMapTasks(100);
-
+
conf.setMapperClass(BayesTfIdfMapper.class);
conf.setInputFormat(SequenceFileInputFormat.class);
conf.setCombinerClass(BayesTfIdfReducer.class);
- conf.setReducerClass(BayesTfIdfReducer.class);
+ conf.setReducerClass(BayesTfIdfReducer.class);
conf.setOutputFormat(BayesTfIdfOutputFormat.class);
-
+
conf.set("io.serializations",
- "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
// Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
- Path interimFile = new Path(output+"/trainer-docCount/part-*");
+ Path interimFile = new Path(output + "/trainer-docCount/part-*");
- Map<String,Double> labelDocumentCounts = SequenceFileModelReader.readLabelDocumentCounts(dfs, interimFile, conf);
+ Map<String, Double> labelDocumentCounts = SequenceFileModelReader.readLabelDocumentCounts(dfs, interimFile, conf);
- DefaultStringifier<Map<String,Double>> mapStringifier =
- new DefaultStringifier<Map<String,Double>>(conf,GenericsUtil.getClass(labelDocumentCounts));
+ DefaultStringifier<Map<String, Double>> mapStringifier =
+ new DefaultStringifier<Map<String, Double>>(conf, GenericsUtil.getClass(labelDocumentCounts));
String labelDocumentCountString = mapStringifier.toString(labelDocumentCounts);
log.info("Counts of documents in Each Label");
- Map<String,Double> c = mapStringifier.fromString(labelDocumentCountString);
+ Map<String, Double> c = mapStringifier.fromString(labelDocumentCountString);
log.info("{}", c);
conf.set("cnaivebayes.labelDocumentCounts", labelDocumentCountString);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java Fri Jul 10 09:35:19 2009
@@ -36,40 +36,35 @@
public class BayesTfIdfMapper extends MapReduceBase implements
Mapper<Text, DoubleWritable, Text, DoubleWritable> {
- private static final Logger log = LoggerFactory.getLogger(BayesTfIdfMapper.class);
+ private static final Logger log = LoggerFactory.getLogger(BayesTfIdfMapper.class);
- private Map<String,Double> labelDocumentCounts = null;
+ private Map<String, Double> labelDocumentCounts = null;
/**
* We need to calculate the Tf-Idf of each feature in each label
- *
- * @param key The label,feature pair (can either be the freq Count or the term
- * Document count
- * @param value
- * @param output
- * @param reporter
- * @throws IOException
+ *
+ * @param key The label,feature pair (can either be the freq Count or the term Document count
*/
@Override
public void map(Text key, DoubleWritable value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
-
+
String labelFeaturePair = key.toString();
char firstChar = labelFeaturePair.charAt(0);
if (firstChar == '-') { // if it is the termDocumentCount
labelFeaturePair = labelFeaturePair.substring(1);
String label = labelFeaturePair.split(",")[0];
-
- if(labelDocumentCounts.containsKey(label) == false){
-
+
+ if (labelDocumentCounts.containsKey(label) == false) {
+
throw new IOException(label);
}
-
+
double labelDocumentCount = labelDocumentCounts.get(label);
double logIdf = Math.log(labelDocumentCount / value.get());
-
+
output.collect(new Text(labelFeaturePair), new DoubleWritable(logIdf));
} else if (firstChar == ',') {
output.collect(new Text("*vocabCount"), new DoubleWritable(1.0));
@@ -77,22 +72,22 @@
output.collect(key, value);
}
}
-
+
@Override
public void configure(JobConf job) {
try {
- if (labelDocumentCounts == null){
- labelDocumentCounts = new HashMap<String,Double>();
+ if (labelDocumentCounts == null) {
+ labelDocumentCounts = new HashMap<String, Double>();
- DefaultStringifier<Map<String,Double>> mapStringifier =
- new DefaultStringifier<Map<String,Double>>(job,GenericsUtil.getClass(labelDocumentCounts));
+ DefaultStringifier<Map<String, Double>> mapStringifier =
+ new DefaultStringifier<Map<String, Double>>(job, GenericsUtil.getClass(labelDocumentCounts));
String labelDocumentCountString = mapStringifier.toString(labelDocumentCounts);
labelDocumentCountString = job.get("cnaivebayes.labelDocumentCounts", labelDocumentCountString);
-
+
labelDocumentCounts = mapStringifier.fromString(labelDocumentCountString);
}
- } catch(IOException ex){
+ } catch (IOException ex) {
log.warn(ex.toString(), ex);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java Fri Jul 10 09:35:19 2009
@@ -31,22 +31,23 @@
import java.io.IOException;
/**
- * This class extends the MultipleOutputFormat, allowing to write the output data to
- * different output files in sequence file output format.
+ * This class extends the MultipleOutputFormat, allowing to write the output data to different output files in sequence
+ * file output format.
*/
-public class BayesTfIdfOutputFormat extends MultipleOutputFormat<WritableComparable<?>,Writable> {
+public class BayesTfIdfOutputFormat extends MultipleOutputFormat<WritableComparable<?>, Writable> {
- private SequenceFileOutputFormat<WritableComparable<?>,Writable> theSequenceFileOutputFormat = null;
+ private SequenceFileOutputFormat<WritableComparable<?>, Writable> theSequenceFileOutputFormat = null;
@Override
protected RecordWriter<WritableComparable<?>, Writable> getBaseRecordWriter(
FileSystem fs, JobConf job, String name, Progressable arg3)
throws IOException {
if (theSequenceFileOutputFormat == null) {
- theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable<?>,Writable>();
+ theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable<?>, Writable>();
}
return theSequenceFileOutputFormat.getRecordWriter(fs, job, name, arg3);
}
+
@Override
protected String generateFileNameForKeyValue(WritableComparable<?> k, Writable v, String name) {
Text key = (Text) k;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java Fri Jul 10 09:35:19 2009
@@ -29,9 +29,7 @@
import java.io.IOException;
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner beacuse only two values should be there inside the values
- */
+/** Can also be used as a local Combiner beacuse only two values should be there inside the values */
public class BayesTfIdfReducer extends MapReduceBase implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
private static final Logger log = LoggerFactory.getLogger(BayesTfIdfReducer.class);
@@ -42,8 +40,8 @@
OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the number of times we've seen this label word per local node. Output is the same
- String token = key.toString();
- if(token.startsWith("*vocabCount")) {
+ String token = key.toString();
+ if (token.startsWith("*vocabCount")) {
double vocabCount = 0.0;
while (values.hasNext()) {
vocabCount += values.next().get();
@@ -58,7 +56,7 @@
//numberofValues ++;
}
//if(numberofValues!=2) throw new IOException("Number of values should be exactly 2");
-
+
output.collect(key, new DoubleWritable(idfTimes_D_ij));
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java Fri Jul 10 09:35:19 2009
@@ -21,28 +21,24 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
import java.io.IOException;
-/**
- * Create and run the Bayes Trainer.
- */
+/** Create and run the Bayes Trainer. */
public class BayesWeightSummerDriver {
private BayesWeightSummerDriver() {
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the the interim files as a
- * {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the the interim files as a {@link
+ * org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -55,13 +51,13 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(BayesWeightSummerDriver.class);
-
+
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(DoubleWritable.class);
@@ -75,11 +71,12 @@
//see the javadoc for the spec for file input formats: first token is key, rest is input. Whole document on one line
conf.setInputFormat(SequenceFileInputFormat.class);
conf.setCombinerClass(BayesWeightSummerReducer.class);
- conf.setReducerClass(BayesWeightSummerReducer.class);
+ conf.setReducerClass(BayesWeightSummerReducer.class);
conf.setOutputFormat(BayesWeightSummerOutputFormat.class);
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
client.setConf(conf);
JobClient.runJob(conf);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java Fri Jul 10 09:35:19 2009
@@ -26,36 +26,28 @@
import java.io.IOException;
-
public class BayesWeightSummerMapper extends MapReduceBase implements
Mapper<Text, DoubleWritable, Text, DoubleWritable> {
-
/**
* We need to calculate the idf of each feature in each label
- *
- * @param key The label,feature pair (can either be the freq Count or the term
- * Document count
- * @param value
- * @param output
- * @param reporter
- * @throws IOException
+ *
+ * @param key The label,feature pair (can either be the freq Count or the term Document count
*/
@Override
public void map(Text key, DoubleWritable value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
int i = labelFeaturePair.indexOf(',');
-
- String label = labelFeaturePair.substring(0,i);
- String feature = labelFeaturePair.substring(i+1);
-
+
+ String label = labelFeaturePair.substring(0, i);
+ String feature = labelFeaturePair.substring(i + 1);
+
output.collect(new Text(',' + feature), value);//sum of weight for all labels for a feature Sigma_j
output.collect(new Text('_' + label), value);//sum of weight for all features for a label Sigma_k
output.collect(new Text("*"), value);//sum of weight of all features for all label Sigma_kSigma_j
-
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java Fri Jul 10 09:35:19 2009
@@ -30,26 +30,26 @@
import java.io.IOException;
/**
- * This class extends the MultipleOutputFormat, allowing to write the output data to
- * different output files in sequence file output format.
+ * This class extends the MultipleOutputFormat, allowing to write the output data to different output files in sequence
+ * file output format.
*/
-public class BayesWeightSummerOutputFormat extends MultipleOutputFormat<WritableComparable<?>,Writable> {
+public class BayesWeightSummerOutputFormat extends MultipleOutputFormat<WritableComparable<?>, Writable> {
- private SequenceFileOutputFormat<WritableComparable<?>,Writable> theSequenceFileOutputFormat = null;
+ private SequenceFileOutputFormat<WritableComparable<?>, Writable> theSequenceFileOutputFormat = null;
@Override
protected RecordWriter<WritableComparable<?>, Writable> getBaseRecordWriter(
FileSystem fs, JobConf job, String name, Progressable arg3)
throws IOException {
if (theSequenceFileOutputFormat == null) {
- theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable<?>,Writable>();
+ theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable<?>, Writable>();
}
return theSequenceFileOutputFormat.getRecordWriter(fs, job, name, arg3);
}
@Override
protected String generateFileNameForKeyValue(WritableComparable<?> k, Writable v,
- String name) {
+ String name) {
Text key = (Text) k;
char firstChar = key.toString().charAt(0);
@@ -58,7 +58,7 @@
} else if (firstChar == ',') { //sum of weight for all labels for a feature Sigma_j
return "Sigma_j/" + name;
} else if (firstChar == '_') { //sum of weights for all features for a label Sigma_k
- return "Sigma_k/"+name;
+ return "Sigma_k/" + name;
}
return "JunkFileThisShouldNotHappen";
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java Fri Jul 10 09:35:19 2009
@@ -28,9 +28,7 @@
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner
- */
+/** Can also be used as a local Combiner */
public class BayesWeightSummerReducer extends MapReduceBase
implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java Fri Jul 10 09:35:19 2009
@@ -35,26 +35,27 @@
import java.util.Map;
/**
- * This Class reads the different interim files created during the Training stage
- * as well as the Model File during testing.
+ * This Class reads the different interim files created during the Training stage as well as the Model File during
+ * testing.
*/
public class SequenceFileModelReader {
private static final Logger log = LoggerFactory.getLogger(SequenceFileModelReader.class);
- private SequenceFileModelReader() {}
+ private SequenceFileModelReader() {
+ }
public static void loadModel(Model model, FileSystem fs, Map<String, Path> pathPatterns,
- Configuration conf) throws IOException {
+ Configuration conf) throws IOException {
loadFeatureWeights(model, fs, pathPatterns.get("sigma_j"), conf);
- loadLabelWeights(model, fs, pathPatterns.get("sigma_k"), conf);
- loadSumWeight(model, fs, pathPatterns.get("sigma_kSigma_j"), conf);
- loadThetaNormalizer(model, fs, pathPatterns.get("thetaNormalizer"), conf);
-
-
+ loadLabelWeights(model, fs, pathPatterns.get("sigma_k"), conf);
+ loadSumWeight(model, fs, pathPatterns.get("sigma_kSigma_j"), conf);
+ loadThetaNormalizer(model, fs, pathPatterns.get("thetaNormalizer"), conf);
+
+
model.initializeWeightMatrix();
-
+
loadWeightMatrix(model, fs, pathPatterns.get("weight"), conf);
model.initializeNormalizer();
//model.GenerateComplementaryModel();
@@ -85,11 +86,11 @@
}
public static void loadFeatureWeights(Model model, FileSystem fs, Path pathPattern,
- Configuration conf) throws IOException {
+ Configuration conf) throws IOException {
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
-
+
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
Path path = fileStatus.getPath();
@@ -108,8 +109,8 @@
}
}
- public static void loadLabelWeights(Model model,FileSystem fs, Path pathPattern,
- Configuration conf) throws IOException {
+ public static void loadLabelWeights(Model model, FileSystem fs, Path pathPattern,
+ Configuration conf) throws IOException {
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
@@ -130,9 +131,9 @@
}
}
}
-
- public static void loadThetaNormalizer(Model model,FileSystem fs, Path pathPattern,
- Configuration conf) throws IOException {
+
+ public static void loadThetaNormalizer(Model model, FileSystem fs, Path pathPattern,
+ Configuration conf) throws IOException {
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
@@ -144,7 +145,7 @@
// the key is either _label_ or label,feature
while (reader.next(key, value)) {
- String keyStr = key.toString();
+ String keyStr = key.toString();
if (keyStr.charAt(0) == '_') { // Sum of weights in a Label
model.setThetaNormalizer(keyStr.substring(1), value
.get());
@@ -154,7 +155,7 @@
}
public static void loadSumWeight(Model model, FileSystem fs, Path pathPattern,
- Configuration conf) throws IOException {
+ Configuration conf) throws IOException {
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
@@ -212,13 +213,13 @@
}
}
- public static Map<String,Double> readLabelSums(FileSystem fs, Path pathPattern, Configuration conf) throws IOException {
- Map<String,Double> labelSum = new HashMap<String,Double>();
+ public static Map<String, Double> readLabelSums(FileSystem fs, Path pathPattern, Configuration conf) throws IOException {
+ Map<String, Double> labelSum = new HashMap<String, Double>();
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
-
+
for (FileStatus fileStatus : outputFiles) {
Path path = fileStatus.getPath();
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, conf);
@@ -235,9 +236,9 @@
return labelSum;
}
- public static Map<String,Double> readLabelDocumentCounts(FileSystem fs, Path pathPattern, Configuration conf)
+ public static Map<String, Double> readLabelDocumentCounts(FileSystem fs, Path pathPattern, Configuration conf)
throws IOException {
- Map<String,Double> labelDocumentCounts = new HashMap<String,Double>();
+ Map<String, Double> labelDocumentCounts = new HashMap<String, Double>();
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
@@ -259,8 +260,8 @@
}
public static double readSigma_jSigma_k(FileSystem fs, Path pathPattern,
- Configuration conf) throws IOException {
- Map<String,Double> weightSum = new HashMap<String,Double>();
+ Configuration conf) throws IOException {
+ Map<String, Double> weightSum = new HashMap<String, Double>();
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
@@ -284,8 +285,8 @@
}
public static double readVocabCount(FileSystem fs, Path pathPattern,
- Configuration conf) throws IOException {
- Map<String,Double> weightSum = new HashMap<String,Double>();
+ Configuration conf) throws IOException {
+ Map<String, Double> weightSum = new HashMap<String, Double>();
Writable key = new Text();
DoubleWritable value = new DoubleWritable();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/package.html
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/package.html?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/package.html (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/package.html Fri Jul 10 09:35:19 2009
@@ -47,9 +47,9 @@
<li>{@link org.apache.mahout.classifier.bayes.BayesDriver} -- Creates the Hadoop Naive Bayes job and outputs
the model. This Driver encapsulates a lot of intermediate Map-Reduce Classes
</li>
- <li>{@link org.apache.mahout.classifier.bayes.common.BayesFeatureDriver}
+ <li>{@link org.apache.mahout.classifier.bayes.common.BayesFeatureDriver}
</li>
- <li>{@link org.apache.mahout.classifier.bayes.common.BayesTfIdfDriver}
+ <li>{@link org.apache.mahout.classifier.bayes.common.BayesTfIdfDriver}
</li>
<li>{@link org.apache.mahout.classifier.bayes.common.BayesWeightSummerDriver}
</li>
@@ -58,7 +58,8 @@
</ol>
The trainer assumes that the input files are in the {@link org.apache.hadoop.mapred.KeyValueTextInputFormat}, i.e.
the first token of the line
- is the label and separated from the remaining tokens on the line by a tab-delimiter. The remaining tokens are the unique features (words). Thus, input documents might look
+ is the label and separated from the remaining tokens on the line by a tab-delimiter. The remaining tokens are the
+ unique features (words). Thus, input documents might look
like:
<pre>
hockey puck stick goalie forward defenseman referee ice checking slapshot helmet
@@ -69,10 +70,13 @@
<p>The output from the trainer is a {@link org.apache.hadoop.io.SequenceFile}.</p>
</div>
<div><h3>The Model</h3>
- <p>The {@link org.apache.mahout.classifier.bayes.BayesModel} is the data structure used to represent the results of the training
- for use by the {@link org.apache.mahout.classifier.bayes.BayesClassifier}. A Model can be created by hand, or, if using
- the {@link org.apache.mahout.classifier.bayes.BayesDriver}, it can be created from the {@link
- org.apache.hadoop.io.SequenceFile} that is output. To create it from the SequenceFile, use the
+
+ <p>The {@link org.apache.mahout.classifier.bayes.BayesModel} is the data structure used to represent the results of
+ the training
+ for use by the {@link org.apache.mahout.classifier.bayes.BayesClassifier}. A Model can be created by hand, or, if
+ using
+ the {@link org.apache.mahout.classifier.bayes.BayesDriver}, it can be created from the {@link
+ org.apache.hadoop.io.SequenceFile} that is output. To create it from the SequenceFile, use the
{@link org.apache.mahout.classifier.bayes.io.SequenceFileModelReader} located in the io subpackage.</p>
</div>
<div><h3>The Classifier</h3>
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java Fri Jul 10 09:35:19 2009
@@ -23,15 +23,13 @@
import org.apache.mahout.common.Model;
import java.util.Collection;
+import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
-import java.util.Deque;
-/**
- * Classifies documents based on a {@link CBayesModel}.
- */
-public class CBayesClassifier implements Classifier{
+/** Classifies documents based on a {@link CBayesModel}. */
+public class CBayesClassifier implements Classifier {
/**
* Classify the document and return the top <code>numResults</code>
@@ -39,8 +37,8 @@
* @param model The model
* @param document The document to classify
* @param defaultCategory The default category to assign
- * @param numResults The maximum number of results to return, ranked by score.
- * Ties are broken by comparing the category
+ * @param numResults The maximum number of results to return, ranked by score. Ties are broken by comparing the
+ * category
* @return A Collection of {@link org.apache.mahout.classifier.ClassifierResult}s.
*/
@Override
@@ -48,7 +46,7 @@
Collection<String> categories = model.getLabels();
PriorityQueue<ClassifierResult> pq = new ClassifierResultPriorityQueue(numResults);
ClassifierResult tmp;
- for (String category : categories){
+ for (String category : categories) {
double prob = documentWeight(model, category, document);
if (prob < 0.0) {
tmp = new ClassifierResult(category, prob);
@@ -60,7 +58,7 @@
while ((tmp = pq.pop()) != null) {
result.addLast(tmp);
}
- if (result.isEmpty()){
+ if (result.isEmpty()) {
result.add(new ClassifierResult(defaultCategory, 0));
}
return result;
@@ -92,12 +90,12 @@
}
/**
- * Calculate the document weight as the multiplication of the
- * {@link Model#featureWeight(String, String)} for each word given the label
+ * Calculate the document weight as the multiplication of the {@link Model#featureWeight(String, String)} for each
+ * word given the label
*
- * @param model The {@link org.apache.mahout.common.Model}
- * @param label The label to calculate the probability of
- * @param document The document
+ * @param model The {@link org.apache.mahout.common.Model}
+ * @param label The label to calculate the probability of
+ * @param document The document
* @return The probability
* @see Model#featureWeight(String, String)
*/
@@ -107,7 +105,7 @@
for (String word : document) {
int[] count = wordList.get(word);
if (count == null) {
- count = new int[] { 0 };
+ count = new int[]{0};
wordList.put(word, count);
}
count[0]++;
@@ -121,7 +119,7 @@
return result;
}
-
+
private static class ClassifierResultPriorityQueue extends PriorityQueue<ClassifierResult> {
private ClassifierResultPriorityQueue(int numResults) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesDriver.java Fri Jul 10 09:35:19 2009
@@ -28,9 +28,7 @@
import java.io.IOException;
-/**
- * Create and run the Bayes Trainer.
- */
+/** Create and run the Bayes Trainer. */
public class CBayesDriver {
private static final Logger log = LoggerFactory.getLogger(CBayesDriver.class);
@@ -39,15 +37,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents
- * live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the
- * {@link org.apache.mahout.common.Model} as a
- * {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
- *
+ * Takes in two arguments: <ol> <li>The input {@link Path} where the input documents live</li>
+ * <li>The output {@link Path} where to write the {@link org.apache.mahout.common.Model} as a
+ * {@link org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -59,16 +52,17 @@
/**
* Run the job
- *
- * @param input the input pathname String
+ *
+ * @param input the input pathname String
* @param output the output pathname String
*/
public static void runJob(String input, String output, int gramSize) throws IOException {
JobConf conf = new JobConf(CBayesDriver.class);
Path outPath = new Path(output);
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
log.info("Reading features...");
//Read the features in each document normalized by length of each document
@@ -92,30 +86,36 @@
//Calculate the normalization factor Sigma_W_ij for each complement class.
//CBayesNormalizedWeightDriver.runJob(input, output);
- Path docCountOutPath = new Path(output+ "/trainer-docCount");
- if (dfs.exists(docCountOutPath))
+ Path docCountOutPath = new Path(output + "/trainer-docCount");
+ if (dfs.exists(docCountOutPath)) {
dfs.delete(docCountOutPath, true);
- Path termDocCountOutPath = new Path(output+ "/trainer-termDocCount");
- if (dfs.exists(termDocCountOutPath))
+ }
+ Path termDocCountOutPath = new Path(output + "/trainer-termDocCount");
+ if (dfs.exists(termDocCountOutPath)) {
dfs.delete(termDocCountOutPath, true);
- Path featureCountOutPath = new Path(output+ "/trainer-featureCount");
- if (dfs.exists(featureCountOutPath))
+ }
+ Path featureCountOutPath = new Path(output + "/trainer-featureCount");
+ if (dfs.exists(featureCountOutPath)) {
dfs.delete(featureCountOutPath, true);
- Path wordFreqOutPath = new Path(output+ "/trainer-wordFreq");
- if (dfs.exists(wordFreqOutPath))
+ }
+ Path wordFreqOutPath = new Path(output + "/trainer-wordFreq");
+ if (dfs.exists(wordFreqOutPath)) {
dfs.delete(wordFreqOutPath, true);
- Path vocabCountPath = new Path(output+ "/trainer-tfIdf/trainer-vocabCount");
- if (dfs.exists(vocabCountPath))
+ }
+ Path vocabCountPath = new Path(output + "/trainer-tfIdf/trainer-vocabCount");
+ if (dfs.exists(vocabCountPath)) {
dfs.delete(vocabCountPath, true);
+ }
/*Path tfIdfOutPath = new Path(output+ "/trainer-tfIdf");
if (dfs.exists(tfIdfOutPath))
dfs.delete(tfIdfOutPath, true);*/
- Path vocabCountOutPath = new Path(output+ "/trainer-vocabCount");
- if (dfs.exists(vocabCountOutPath))
+ Path vocabCountOutPath = new Path(output + "/trainer-vocabCount");
+ if (dfs.exists(vocabCountOutPath)) {
dfs.delete(vocabCountOutPath, true);
- /* Path weightsOutPath = new Path(output+ "/trainer-weights");
- if (dfs.exists(weightsOutPath))
- dfs.delete(weightsOutPath, true);*/
+ }
+ /* Path weightsOutPath = new Path(output+ "/trainer-weights");
+ if (dfs.exists(weightsOutPath))
+ dfs.delete(weightsOutPath, true);*/
/*Path thetaOutPath = new Path(output+ "/trainer-theta");
if (dfs.exists(thetaOutPath))
dfs.delete(thetaOutPath, true);*/
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesModel.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesModel.java Fri Jul 10 09:35:19 2009
@@ -30,7 +30,7 @@
@Override
protected double getWeight(Integer label, Integer feature) {
double result = 0.0;
- Map<Integer,Double> featureWeights = featureLabelWeights.get(feature);
+ Map<Integer, Double> featureWeights = featureLabelWeights.get(feature);
if (featureWeights.containsKey(label)) {
result = featureWeights.get(label);
@@ -41,8 +41,8 @@
double numerator = sigma_j - result + alpha_i;
double denominator = (sigma_jSigma_k - sumLabelWeight + vocabCount);
-
- double weight = Math.log(numerator /denominator);
+
+ double weight = Math.log(numerator / denominator);
result = -weight / getThetaNormalizer(label);
return result;
}
@@ -50,7 +50,7 @@
@Override
protected double getWeightUnprocessed(Integer label, Integer feature) {
double result;
- Map<Integer,Double> featureWeights = featureLabelWeights.get(feature);
+ Map<Integer, Double> featureWeights = featureLabelWeights.get(feature);
if (featureWeights.containsKey(label)) {
result = featureWeights.get(label);
@@ -79,105 +79,107 @@
/ perLabelWeightSumNormalisationFactor);
}
log.info("{}", thetaNormalizer);
-
- /*for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- thetaNormalizer.put(label, 0.0);
- }
- for (int feature = 0, maxFeatures = featureList.size(); feature < maxFeatures; feature++) {
- for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
-
- double D_ij = getWeightUnprocessed(label, feature);
- double sumLabelWeight = getSumLabelWeight(label);
- double sigma_j = getSumFeatureWeight(feature);
- double vocabCount = featureList.size();
-
- double numerator = (sigma_j ) + alpha_i;
- double denominator = (sigma_jSigma_k - sumLabelWeight + vocabCount);
- double denominator1 = 0.5 *(sigma_jSigma_k/vocabCount + D_ij * (double) maxLabels);
- double weight = Math.log(numerator / denominator) + Math.log( 1 - D_ij/denominator1 );
-
- thetaNormalizer.put(label, weight+thetaNormalizer.get(label));
-
- }
- }
- perLabelWeightSumNormalisationFactor = Double.MAX_VALUE;
- log.info("{}", thetaNormalizer);
- for (Integer label : thetaNormalizer.keySet()) {
- double Sigma_W_ij = thetaNormalizer.get(label);
- if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
- perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
- }
- }
- for (Integer label : thetaNormalizer.keySet()) {
- double Sigma_W_ij = thetaNormalizer.get(label);
- thetaNormalizer.put(label, Sigma_W_ij
- / perLabelWeightSumNormalisationFactor);
- }
- log.info("{}", thetaNormalizer);*/
+ /*for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
+ thetaNormalizer.put(label, 0.0);
+ }
+ for (int feature = 0, maxFeatures = featureList.size(); feature < maxFeatures; feature++) {
+ for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
+
+ double D_ij = getWeightUnprocessed(label, feature);
+ double sumLabelWeight = getSumLabelWeight(label);
+ double sigma_j = getSumFeatureWeight(feature);
+ double vocabCount = featureList.size();
+
+ double numerator = (sigma_j ) + alpha_i;
+ double denominator = (sigma_jSigma_k - sumLabelWeight + vocabCount);
+ double denominator1 = 0.5 *(sigma_jSigma_k/vocabCount + D_ij * (double) maxLabels);
+ double weight = Math.log(numerator / denominator) + Math.log( 1 - D_ij/denominator1 );
+
+ thetaNormalizer.put(label, weight+thetaNormalizer.get(label));
+
+ }
+ }
+ perLabelWeightSumNormalisationFactor = Double.MAX_VALUE;
+ log.info("{}", thetaNormalizer);
+ for (Integer label : thetaNormalizer.keySet()) {
+ double Sigma_W_ij = thetaNormalizer.get(label);
+ if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
+ perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
+ }
+ }
+
+ for (Integer label : thetaNormalizer.keySet()) {
+ double Sigma_W_ij = thetaNormalizer.get(label);
+ thetaNormalizer.put(label, Sigma_W_ij
+ / perLabelWeightSumNormalisationFactor);
+ }
+ log.info("{}", thetaNormalizer);*/
}
@Override
public void generateModel() {
- double vocabCount = featureList.size();
+ double vocabCount = featureList.size();
- double[] perLabelThetaNormalizer = new double[labelList.size()];
+ double[] perLabelThetaNormalizer = new double[labelList.size()];
for (int feature = 0, maxFeatures = featureList.size(); feature < maxFeatures; feature++) {
- Integer featureInt = feature;
- for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
+ Integer featureInt = feature;
+ for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- Integer labelInt = label;
- double D_ij = getWeightUnprocessed(labelInt, featureInt);
- double sumLabelWeight = getSumLabelWeight(labelInt);
- double sigma_j = getSumFeatureWeight(featureInt);
+ Integer labelInt = label;
+ double D_ij = getWeightUnprocessed(labelInt, featureInt);
+ double sumLabelWeight = getSumLabelWeight(labelInt);
+ double sigma_j = getSumFeatureWeight(featureInt);
- double numerator = (sigma_j - D_ij) + alpha_i;
- double denominator = (sigma_jSigma_k - sumLabelWeight) + vocabCount;
+ double numerator = (sigma_j - D_ij) + alpha_i;
+ double denominator = (sigma_jSigma_k - sumLabelWeight) + vocabCount;
- double weight = Math.log(numerator / denominator);
+ double weight = Math.log(numerator / denominator);
- if (D_ij != 0)
- setWeight(labelInt, featureInt, weight);
+ if (D_ij != 0) {
+ setWeight(labelInt, featureInt, weight);
+ }
- perLabelThetaNormalizer[label] += weight;
+ perLabelThetaNormalizer[label] += weight;
- }
}
- log.info("Normalizing Weights");
+ }
+ log.info("Normalizing Weights");
double perLabelWeightSumNormalisationFactor = Double.MAX_VALUE;
for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- double Sigma_W_ij = perLabelThetaNormalizer[label];
- if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
- perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
- }
+ double Sigma_W_ij = perLabelThetaNormalizer[label];
+ if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
+ perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
}
+ }
- for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- double Sigma_W_ij = perLabelThetaNormalizer[label];
- perLabelThetaNormalizer[label] = Sigma_W_ij
- / perLabelWeightSumNormalisationFactor;
- }
+ for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
+ double Sigma_W_ij = perLabelThetaNormalizer[label];
+ perLabelThetaNormalizer[label] = Sigma_W_ij
+ / perLabelWeightSumNormalisationFactor;
+ }
- for (int feature = 0, maxFeatures = featureList.size(); feature < maxFeatures; feature++) {
- Integer featureInt = feature;
- for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- Integer labelInt = label;
- double W_ij = getWeightUnprocessed(labelInt, featureInt);
- if (W_ij == 0)
- continue;
- double Sigma_W_ij = perLabelThetaNormalizer[label];
- double normalizedWeight = -1.0 * (W_ij / Sigma_W_ij);
- setWeight(labelInt, featureInt, normalizedWeight);
+ for (int feature = 0, maxFeatures = featureList.size(); feature < maxFeatures; feature++) {
+ Integer featureInt = feature;
+ for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
+ Integer labelInt = label;
+ double W_ij = getWeightUnprocessed(labelInt, featureInt);
+ if (W_ij == 0) {
+ continue;
}
+ double Sigma_W_ij = perLabelThetaNormalizer[label];
+ double normalizedWeight = -1.0 * (W_ij / Sigma_W_ij);
+ setWeight(labelInt, featureInt, normalizedWeight);
}
+ }
}
-
+
/**
* Get the weighted probability of the feature.
- *
- * @param label The label of the feature
+ *
+ * @param label The label of the feature
* @param feature The feature to calc. the prob. for
* @return The weighted probability
*/
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightDriver.java Fri Jul 10 09:35:19 2009
@@ -22,23 +22,21 @@
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.util.GenericsUtil;
import org.apache.mahout.classifier.bayes.io.SequenceFileModelReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
import java.io.IOException;
+import java.util.Map;
-/**
- * Create and run the Bayes Trainer.
- */
+/** Create and run the Bayes Trainer. */
public class CBayesNormalizedWeightDriver {
private static final Logger log = LoggerFactory.getLogger(CBayesNormalizedWeightDriver.class);
@@ -47,12 +45,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the {@link org.apache.mahout.common.Model}
- * as a {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the {@link org.apache.mahout.common.Model} as a
+ * {@link org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -65,13 +61,13 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(CBayesNormalizedWeightDriver.class);
-
+
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(DoubleWritable.class);
@@ -82,41 +78,40 @@
//conf.setNumReduceTasks(1);
conf.setMapperClass(CBayesNormalizedWeightMapper.class);
conf.setInputFormat(SequenceFileInputFormat.class);
- conf.setCombinerClass(CBayesNormalizedWeightReducer.class);
- conf.setReducerClass(CBayesNormalizedWeightReducer.class);
+ conf.setCombinerClass(CBayesNormalizedWeightReducer.class);
+ conf.setReducerClass(CBayesNormalizedWeightReducer.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
-
+
conf.set("io.serializations",
- "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
// Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
- Path thetaNormalizationsFiles = new Path(output+"/trainer-thetaNormalizer/part*");
- Map<String,Double> thetaNormalizer= SequenceFileModelReader.readLabelSums(dfs, thetaNormalizationsFiles, conf);
+ Path thetaNormalizationsFiles = new Path(output + "/trainer-thetaNormalizer/part*");
+ Map<String, Double> thetaNormalizer = SequenceFileModelReader.readLabelSums(dfs, thetaNormalizationsFiles, conf);
double perLabelWeightSumNormalisationFactor = Double.MAX_VALUE;
- for(Map.Entry<String, Double> stringDoubleEntry1 : thetaNormalizer.entrySet())
- {
+ for (Map.Entry<String, Double> stringDoubleEntry1 : thetaNormalizer.entrySet()) {
double Sigma_W_ij = stringDoubleEntry1.getValue();
- if(perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)){
+ if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
}
}
- for(Map.Entry<String, Double> stringDoubleEntry : thetaNormalizer.entrySet())
- {
+ for (Map.Entry<String, Double> stringDoubleEntry : thetaNormalizer.entrySet()) {
double Sigma_W_ij = stringDoubleEntry.getValue();
- thetaNormalizer.put(stringDoubleEntry.getKey(), Sigma_W_ij / perLabelWeightSumNormalisationFactor) ;
+ thetaNormalizer.put(stringDoubleEntry.getKey(), Sigma_W_ij / perLabelWeightSumNormalisationFactor);
}
- DefaultStringifier<Map<String,Double>> mapStringifier =
- new DefaultStringifier<Map<String,Double>>(conf, GenericsUtil.getClass(thetaNormalizer));
+ DefaultStringifier<Map<String, Double>> mapStringifier =
+ new DefaultStringifier<Map<String, Double>>(conf, GenericsUtil.getClass(thetaNormalizer));
String thetaNormalizationsString = mapStringifier.toString(thetaNormalizer);
- Map<String,Double> c = mapStringifier.fromString(thetaNormalizationsString);
+ Map<String, Double> c = mapStringifier.fromString(thetaNormalizationsString);
log.info("{}", c);
conf.set("cnaivebayes.thetaNormalizations", thetaNormalizationsString);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightMapper.java Fri Jul 10 09:35:19 2009
@@ -36,23 +36,18 @@
public class CBayesNormalizedWeightMapper extends MapReduceBase implements
Mapper<Text, DoubleWritable, Text, DoubleWritable> {
- private static final Logger log = LoggerFactory.getLogger(CBayesNormalizedWeightMapper.class);
+ private static final Logger log = LoggerFactory.getLogger(CBayesNormalizedWeightMapper.class);
- private Map<String,Double> thetaNormalizer = null;
+ private Map<String, Double> thetaNormalizer = null;
/**
* We need to calculate the idf of each feature in each label
- *
- * @param key The label,feature pair (can either be the freq Count or the term
- * Document count
- * @param value
- * @param output
- * @param reporter
- * @throws IOException
+ *
+ * @param key The label,feature pair (can either be the freq Count or the term Document count
*/
@Override
public void map(Text key, DoubleWritable value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
@@ -66,9 +61,9 @@
public void configure(JobConf job) {
try {
if (thetaNormalizer == null) {
- thetaNormalizer = new HashMap<String,Double>();
+ thetaNormalizer = new HashMap<String, Double>();
- DefaultStringifier<Map<String,Double>> mapStringifier = new DefaultStringifier<Map<String,Double>>(
+ DefaultStringifier<Map<String, Double>> mapStringifier = new DefaultStringifier<Map<String, Double>>(
job, GenericsUtil.getClass(thetaNormalizer));
String thetaNormalizationsString = mapStringifier.toString(thetaNormalizer);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightReducer.java Fri Jul 10 09:35:19 2009
@@ -29,13 +29,11 @@
import java.io.IOException;
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner beacuse only two values should be there inside the values
- */
+/** Can also be used as a local Combiner beacuse only two values should be there inside the values */
public class CBayesNormalizedWeightReducer extends MapReduceBase
implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
- private static final Logger log = LoggerFactory.getLogger(CBayesNormalizedWeightReducer.class);
+ private static final Logger log = LoggerFactory.getLogger(CBayesNormalizedWeightReducer.class);
@Override
public void reduce(Text key,
@@ -43,15 +41,16 @@
OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the number of times we've seen this label word per local node. Output is the same
- String token = key.toString();
+ String token = key.toString();
double weight = 0.0;
while (values.hasNext()) {
weight += values.next().get();
}
- if(token.equalsIgnoreCase("rec.motorcycles,miller"))
+ if (token.equalsIgnoreCase("rec.motorcycles,miller")) {
log.info("{}=>{}", token, weight);
+ }
output.collect(key, new DoubleWritable(weight));
}
-
+
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaDriver.java Fri Jul 10 09:35:19 2009
@@ -22,23 +22,21 @@
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.util.GenericsUtil;
import org.apache.mahout.classifier.bayes.io.SequenceFileModelReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
import java.io.IOException;
+import java.util.Map;
-/**
- * Create and run the Bayes Trainer.
- */
+/** Create and run the Bayes Trainer. */
public class CBayesThetaDriver {
private static final Logger log = LoggerFactory.getLogger(CBayesThetaDriver.class);
@@ -47,12 +45,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
* <li>The output {@link org.apache.hadoop.fs.Path} where to write the {@link org.apache.mahout.common.Model} as a
- * {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * {@link org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -65,13 +61,13 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(CBayesThetaDriver.class);
-
+
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(DoubleWritable.class);
@@ -85,29 +81,30 @@
conf.setMapperClass(CBayesThetaMapper.class);
conf.setInputFormat(SequenceFileInputFormat.class);
//conf.setCombinerClass(CBayesThetaReducer.class);
- conf.setReducerClass(CBayesThetaReducer.class);
+ conf.setReducerClass(CBayesThetaReducer.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
-
+
conf.set("io.serializations",
- "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
// Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
- Path Sigma_kFiles = new Path(output+"/trainer-weights/Sigma_k/*");
- Map<String,Double> labelWeightSum= SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf);
- DefaultStringifier<Map<String,Double>> mapStringifier =
- new DefaultStringifier<Map<String,Double>>(conf, GenericsUtil.getClass(labelWeightSum));
+ Path Sigma_kFiles = new Path(output + "/trainer-weights/Sigma_k/*");
+ Map<String, Double> labelWeightSum = SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf);
+ DefaultStringifier<Map<String, Double>> mapStringifier =
+ new DefaultStringifier<Map<String, Double>>(conf, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
log.info("Sigma_k for Each Label");
- Map<String,Double> c = mapStringifier.fromString(labelWeightSumString);
+ Map<String, Double> c = mapStringifier.fromString(labelWeightSumString);
log.info("{}", c);
conf.set("cnaivebayes.sigma_k", labelWeightSumString);
- Path sigma_kSigma_jFile = new Path(output+"/trainer-weights/Sigma_kSigma_j/*");
+ Path sigma_kSigma_jFile = new Path(output + "/trainer-weights/Sigma_kSigma_j/*");
double sigma_jSigma_k = SequenceFileModelReader.readSigma_jSigma_k(dfs, sigma_kSigma_jFile, conf);
DefaultStringifier<Double> stringifier = new DefaultStringifier<Double>(conf, Double.class);
String sigma_jSigma_kString = stringifier.toString(sigma_jSigma_k);
@@ -117,7 +114,7 @@
log.info("{}", retSigma_jSigma_k);
conf.set("cnaivebayes.sigma_jSigma_k", sigma_jSigma_kString);
- Path vocabCountFile = new Path(output+"/trainer-tfIdf/trainer-vocabCount/*");
+ Path vocabCountFile = new Path(output + "/trainer-tfIdf/trainer-vocabCount/*");
double vocabCount = SequenceFileModelReader.readVocabCount(dfs, vocabCountFile, conf);
String vocabCountString = stringifier.toString(vocabCount);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaMapper.java Fri Jul 10 09:35:19 2009
@@ -36,25 +36,20 @@
public class CBayesThetaMapper extends MapReduceBase implements
Mapper<Text, DoubleWritable, Text, DoubleWritable> {
- private static final Logger log = LoggerFactory.getLogger(CBayesThetaMapper.class);
+ private static final Logger log = LoggerFactory.getLogger(CBayesThetaMapper.class);
- private Map<String,Double> labelWeightSum = null;
+ private Map<String, Double> labelWeightSum = null;
private double sigma_jSigma_k = 0.0;
private double vocabCount = 0.0;
/**
* We need to calculate the idf of each feature in each label
- *
- * @param key The label,feature pair (can either be the freq Count or the term
- * Document count
- * @param value
- * @param output
- * @param reporter
- * @throws IOException
+ *
+ * @param key The label,feature pair (can either be the freq Count or the term Document count
*/
@Override
public void map(Text key, DoubleWritable value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
@@ -64,12 +59,12 @@
double alpha_i = 1.0;
for (Map.Entry<String, Double> stringDoubleEntry : labelWeightSum.entrySet()) {
double inverseDenominator = 1.0 / (sigma_jSigma_k - stringDoubleEntry.getValue() + vocabCount);
- DoubleWritable weight = new DoubleWritable((value.get() + alpha_i)*inverseDenominator);
+ DoubleWritable weight = new DoubleWritable((value.get() + alpha_i) * inverseDenominator);
output.collect(new Text((stringDoubleEntry.getKey() + ',' + feature).trim()), weight); //output Sigma_j
}
} else {
String label = labelFeaturePair.split(",")[0];
- double inverseDenominator = 1.0 /(sigma_jSigma_k - labelWeightSum.get(label) + vocabCount);
+ double inverseDenominator = 1.0 / (sigma_jSigma_k - labelWeightSum.get(label) + vocabCount);
DoubleWritable weight = new DoubleWritable(-value.get() * inverseDenominator);
output.collect(key, weight);//output -D_ij
}
@@ -79,9 +74,9 @@
public void configure(JobConf job) {
try {
if (labelWeightSum == null) {
- labelWeightSum = new HashMap<String,Double>();
+ labelWeightSum = new HashMap<String, Double>();
- DefaultStringifier<Map<String,Double>> mapStringifier = new DefaultStringifier<Map<String,Double>>(
+ DefaultStringifier<Map<String, Double>> mapStringifier = new DefaultStringifier<Map<String, Double>>(
job, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
@@ -100,7 +95,7 @@
vocabCountString = job.get("cnaivebayes.vocabCount",
vocabCountString);
vocabCount = stringifier.fromString(vocabCountString);
-
+
}
} catch (IOException ex) {
log.info(ex.toString(), ex);