You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/07/10 11:35:28 UTC
svn commit: r792856 [7/13] - in /lucene/mahout/trunk/core/src:
main/java/org/apache/mahout/cf/taste/common/
main/java/org/apache/mahout/cf/taste/eval/
main/java/org/apache/mahout/cf/taste/hadoop/
main/java/org/apache/mahout/cf/taste/impl/common/ main/j...
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerDriver.java Fri Jul 10 09:35:19 2009
@@ -22,23 +22,21 @@
import org.apache.hadoop.io.DefaultStringifier;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
-import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.util.GenericsUtil;
import org.apache.mahout.classifier.bayes.io.SequenceFileModelReader;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Map;
import java.io.IOException;
+import java.util.Map;
-/**
- * Create and run the Bayes Trainer.
- */
+/** Create and run the Bayes Trainer. */
public class CBayesThetaNormalizerDriver {
private static final Logger log = LoggerFactory.getLogger(CBayesThetaNormalizerDriver.class);
@@ -47,12 +45,10 @@
}
/**
- * Takes in two arguments:
- * <ol>
- * <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the {@link org.apache.mahout.common.Model}
- * as a {@link org.apache.hadoop.io.SequenceFile}</li>
- * </ol>
+ * Takes in two arguments: <ol> <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the {@link org.apache.mahout.common.Model} as a
+ * {@link org.apache.hadoop.io.SequenceFile}</li> </ol>
+ *
* @param args The args
*/
public static void main(String[] args) throws IOException {
@@ -65,13 +61,13 @@
/**
* Run the job
*
- * @param input the input pathname String
- * @param output the output pathname String
+ * @param input the input pathname String
+ * @param output the output pathname String
*/
public static void runJob(String input, String output) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(CBayesThetaNormalizerDriver.class);
-
+
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(DoubleWritable.class);
@@ -83,30 +79,31 @@
//conf.setNumReduceTasks(1);
conf.setMapperClass(CBayesThetaNormalizerMapper.class);
conf.setInputFormat(SequenceFileInputFormat.class);
- conf.setCombinerClass(CBayesThetaNormalizerReducer.class);
- conf.setReducerClass(CBayesThetaNormalizerReducer.class);
+ conf.setCombinerClass(CBayesThetaNormalizerReducer.class);
+ conf.setReducerClass(CBayesThetaNormalizerReducer.class);
conf.setOutputFormat(SequenceFileOutputFormat.class);
conf.set("io.serializations",
- "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
+ "org.apache.hadoop.io.serializer.JavaSerialization,org.apache.hadoop.io.serializer.WritableSerialization");
// Dont ever forget this. People should keep track of how hadoop conf parameters and make or break a piece of code
-
+
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
- Path Sigma_kFiles = new Path(output+"/trainer-weights/Sigma_k/*");
- Map<String,Double> labelWeightSum= SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf);
- DefaultStringifier<Map<String,Double>> mapStringifier =
- new DefaultStringifier<Map<String,Double>>(conf, GenericsUtil.getClass(labelWeightSum));
+ Path Sigma_kFiles = new Path(output + "/trainer-weights/Sigma_k/*");
+ Map<String, Double> labelWeightSum = SequenceFileModelReader.readLabelSums(dfs, Sigma_kFiles, conf);
+ DefaultStringifier<Map<String, Double>> mapStringifier =
+ new DefaultStringifier<Map<String, Double>>(conf, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
log.info("Sigma_k for Each Label");
- Map<String,Double> c = mapStringifier.fromString(labelWeightSumString);
+ Map<String, Double> c = mapStringifier.fromString(labelWeightSumString);
log.info("{}", c);
conf.set("cnaivebayes.sigma_k", labelWeightSumString);
- Path sigma_kSigma_jFile = new Path(output+"/trainer-weights/Sigma_kSigma_j/*");
+ Path sigma_kSigma_jFile = new Path(output + "/trainer-weights/Sigma_kSigma_j/*");
double sigma_jSigma_k = SequenceFileModelReader.readSigma_jSigma_k(dfs, sigma_kSigma_jFile, conf);
DefaultStringifier<Double> stringifier = new DefaultStringifier<Double>(conf, Double.class);
String sigma_jSigma_kString = stringifier.toString(sigma_jSigma_k);
@@ -116,7 +113,7 @@
log.info("{}", retSigma_jSigma_k);
conf.set("cnaivebayes.sigma_jSigma_k", sigma_jSigma_kString);
- Path vocabCountFile = new Path(output+"/trainer-tfIdf/trainer-vocabCount/*");
+ Path vocabCountFile = new Path(output + "/trainer-tfIdf/trainer-vocabCount/*");
double vocabCount = SequenceFileModelReader.readVocabCount(dfs, vocabCountFile, conf);
String vocabCountString = stringifier.toString(vocabCount);
@@ -128,6 +125,6 @@
client.setConf(conf);
JobClient.runJob(conf);
-
+
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerMapper.java Fri Jul 10 09:35:19 2009
@@ -38,23 +38,18 @@
private static final Logger log = LoggerFactory.getLogger(CBayesThetaNormalizerMapper.class);
- private Map<String,Double> labelWeightSum = null;
+ private Map<String, Double> labelWeightSum = null;
private double sigma_jSigma_k = 0.0;
private double vocabCount = 0.0;
/**
* We need to calculate the idf of each feature in each label
- *
- * @param key The label,feature pair (can either be the freq Count or the term
- * Document count
- * @param value
- * @param output
- * @param reporter
- * @throws IOException
+ *
+ * @param key The label,feature pair (can either be the freq Count or the term Document count
*/
@Override
public void map(Text key, DoubleWritable value,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
@@ -62,31 +57,30 @@
double alpha_i = 1.0;
for (Map.Entry<String, Double> stringDoubleEntry : labelWeightSum.entrySet()) {
- double weight = Math.log((value.get() + alpha_i)/(sigma_jSigma_k - stringDoubleEntry.getValue() + vocabCount));
+ double weight = Math.log((value.get() + alpha_i) / (sigma_jSigma_k - stringDoubleEntry.getValue() + vocabCount));
output.collect(new Text(('_' + stringDoubleEntry.getKey()).trim()), new DoubleWritable(weight)); //output Sigma_j
}
-
- }
- else {
+
+ } else {
String label = labelFeaturePair.split(",")[0];
-
+
double D_ij = value.get();
- double denominator = 0.5 *((sigma_jSigma_k / vocabCount) + (D_ij * this.labelWeightSum.size()));
+ double denominator = 0.5 * ((sigma_jSigma_k / vocabCount) + (D_ij * this.labelWeightSum.size()));
double weight = Math.log(1.0 - D_ij / denominator);
- output.collect(new Text(('_' +label).trim()), new DoubleWritable(weight));//output -D_ij
-
+ output.collect(new Text(('_' + label).trim()), new DoubleWritable(weight));//output -D_ij
+
}
-
+
}
@Override
public void configure(JobConf job) {
try {
if (labelWeightSum == null) {
- labelWeightSum = new HashMap<String,Double>();
+ labelWeightSum = new HashMap<String, Double>();
- DefaultStringifier<Map<String,Double>> mapStringifier = new DefaultStringifier<Map<String,Double>>(
+ DefaultStringifier<Map<String, Double>> mapStringifier = new DefaultStringifier<Map<String, Double>>(
job, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerReducer.java Fri Jul 10 09:35:19 2009
@@ -27,16 +27,13 @@
import java.io.IOException;
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner beacuse only two values should be there
- * inside the values
- */
+/** Can also be used as a local Combiner beacuse only two values should be there inside the values */
public class CBayesThetaNormalizerReducer extends MapReduceBase implements
Reducer<Text, DoubleWritable, Text, DoubleWritable> {
@Override
public void reduce(Text key, Iterator<DoubleWritable> values,
- OutputCollector<Text, DoubleWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
// Key is label,word, value is the number of times we've seen this label
// word per local node. Output is the same
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaReducer.java Fri Jul 10 09:35:19 2009
@@ -29,31 +29,32 @@
import java.io.IOException;
import java.util.Iterator;
-/**
- * Can also be used as a local Combiner beacuse only two values should be there inside the values
- */
+/** Can also be used as a local Combiner beacuse only two values should be there inside the values */
public class CBayesThetaReducer extends MapReduceBase implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
-
+
private static final Logger log = LoggerFactory.getLogger(CBayesThetaReducer.class);
-
+
@Override
public void reduce(Text key,
Iterator<DoubleWritable> values,
OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the number of times we've seen this label word per local node. Output is the same
- String token = key.toString();
+ String token = key.toString();
double weight = 0.0;
int numberofValues = 0;
while (values.hasNext()) {
weight += values.next().get();
- numberofValues ++;
- }
- if(numberofValues < 2) return;
- if(weight <= 0.0)
+ numberofValues++;
+ }
+ if (numberofValues < 2) {
+ return;
+ }
+ if (weight <= 0.0) {
log.info("{}=>{}", token, weight);
+ }
output.collect(key, new DoubleWritable(weight));
}
-
+
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java Fri Jul 10 09:35:19 2009
@@ -1,12 +1,12 @@
package org.apache.mahout.clustering;
-import org.apache.mahout.matrix.Vector;
-import org.apache.mahout.matrix.SparseVector;
import org.apache.hadoop.io.Writable;
+import org.apache.mahout.matrix.SparseVector;
+import org.apache.mahout.matrix.Vector;
+import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.io.DataInput;
/**
*
@@ -35,7 +35,7 @@
/**
* Return the center point
- *
+ *
* @return the center of the Cluster
*/
public Vector getCenter() {
@@ -50,21 +50,15 @@
/**
* Simply writes out the id, and that's it!
- *
+ *
* @param out The {@link java.io.DataOutput}
- * @throws IOException
*/
@Override
public void write(DataOutput out) throws IOException {
out.writeInt(id);
}
- /**
- * Reads in the id, nothing else
- *
- * @param in
- * @throws IOException
- */
+ /** Reads in the id, nothing else */
@Override
public void readFields(DataInput in) throws IOException {
id = in.readInt();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java Fri Jul 10 09:35:19 2009
@@ -17,11 +17,6 @@
package org.apache.mahout.clustering.canopy;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.util.List;
-
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
@@ -31,10 +26,14 @@
import org.apache.mahout.matrix.Vector;
import org.apache.mahout.utils.DistanceMeasure;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.List;
+
/**
- * This class models a canopy as a center point, the number of points that are
- * contained within it according to the application of some distance metric, and
- * a point total which is the sum of all the points and is used to compute the
+ * This class models a canopy as a center point, the number of points that are contained within it according to the
+ * application of some distance metric, and a point total which is the sum of all the points and is used to compute the
* centroid when needed.
*/
public class Canopy extends ClusterBase implements Writable {
@@ -61,15 +60,13 @@
private static DistanceMeasure measure;
- /**
- * Used w
- */
+ /** Used w */
public Canopy() {
}
/**
* Create a new Canopy containing the given point
- *
+ *
* @param point a point in vector space
*/
public Canopy(Vector point) {
@@ -81,8 +78,8 @@
/**
* Create a new Canopy containing the given point and canopyId
- *
- * @param point a point in vector space
+ *
+ * @param point a point in vector space
* @param canopyId an int identifying the canopy local to this process only
*/
public Canopy(Vector point, int canopyId) {
@@ -94,7 +91,7 @@
/**
* Configure the Canopy and its distance measure
- *
+ *
* @param job the JobConf for this job
*/
public static void configure(JobConf job) {
@@ -115,13 +112,7 @@
t2 = Double.parseDouble(job.get(T2_KEY));
}
- /**
- * Configure the Canopy for unit tests
- *
- * @param aMeasure
- * @param aT1
- * @param aT2
- */
+ /** Configure the Canopy for unit tests */
public static void config(DistanceMeasure aMeasure, double aT1, double aT2) {
nextCanopyId = 0;
measure = aMeasure;
@@ -130,45 +121,44 @@
}
/**
- * This is the same algorithm as the reference but inverted to iterate over
- * existing canopies instead of the points. Because of this it does not need
- * to actually store the points, instead storing a total points vector and the
- * number of points. From this a centroid can be computed.
- * <p/>
- * This method is used by the CanopyReducer.
- *
- * @param point the point to be added
+ * This is the same algorithm as the reference but inverted to iterate over existing canopies instead of the points.
+ * Because of this it does not need to actually store the points, instead storing a total points vector and the number
+ * of points. From this a centroid can be computed. <p/> This method is used by the CanopyReducer.
+ *
+ * @param point the point to be added
* @param canopies the List<Canopy> to be appended
*/
public static void addPointToCanopies(Vector point, List<Canopy> canopies) {
boolean pointStronglyBound = false;
for (Canopy canopy : canopies) {
double dist = measure.distance(canopy.getCenter().getLengthSquared(), canopy.getCenter(), point);
- if (dist < t1)
+ if (dist < t1) {
canopy.addPoint(point);
+ }
pointStronglyBound = pointStronglyBound || (dist < t2);
}
- if (!pointStronglyBound)
+ if (!pointStronglyBound) {
canopies.add(new Canopy(point));
+ }
}
/**
- * This method is used by the CanopyMapper to perform canopy inclusion tests
- * and to emit the point and its covering canopies to the output. The
- * CanopyCombiner will then sum the canopy points and produce the centroids.
- *
- * @param point the point to be added
- * @param canopies the List<Canopy> to be appended
+ * This method is used by the CanopyMapper to perform canopy inclusion tests and to emit the point and its covering
+ * canopies to the output. The CanopyCombiner will then sum the canopy points and produce the centroids.
+ *
+ * @param point the point to be added
+ * @param canopies the List<Canopy> to be appended
* @param collector an OutputCollector in which to emit the point
*/
public static void emitPointToNewCanopies(Vector point,
- List<Canopy> canopies, OutputCollector<Text, Vector> collector)
+ List<Canopy> canopies, OutputCollector<Text, Vector> collector)
throws IOException {
boolean pointStronglyBound = false;
for (Canopy canopy : canopies) {
double dist = measure.distance(canopy.getCenter().getLengthSquared(), canopy.getCenter(), point);
- if (dist < t1)
+ if (dist < t1) {
canopy.emitPoint(point, collector);
+ }
pointStronglyBound = pointStronglyBound || (dist < t2);
}
if (!pointStronglyBound) {
@@ -179,18 +169,17 @@
}
/**
- * This method is used by the CanopyMapper to perform canopy inclusion tests
- * and to emit the point keyed by its covering canopies to the output. if the
- * point is not covered by any canopies (due to canopy centroid clustering),
+ * This method is used by the CanopyMapper to perform canopy inclusion tests and to emit the point keyed by its
+ * covering canopies to the output. if the point is not covered by any canopies (due to canopy centroid clustering),
* emit the point to the closest covering canopy.
- *
- * @param point the point to be added
- * @param canopies the List<Canopy> to be appended
+ *
+ * @param point the point to be added
+ * @param canopies the List<Canopy> to be appended
* @param collector an OutputCollector in which to emit the point
*/
public static void emitPointToExistingCanopies(Vector point,
- List<Canopy> canopies,
- OutputCollector<Text, Vector> collector) throws IOException {
+ List<Canopy> canopies,
+ OutputCollector<Text, Vector> collector) throws IOException {
double minDist = Double.MAX_VALUE;
Canopy closest = null;
boolean isCovered = false;
@@ -206,8 +195,9 @@
}
// if the point is not contained in any canopies (due to canopy centroid
// clustering), emit the point to the closest covering canopy.
- if (!isCovered)
+ if (!isCovered) {
collector.collect(new Text(closest.getIdentifier()), point);
+ }
}
@@ -225,11 +215,7 @@
this.numPoints = 1;
}
- /**
- * Format the canopy for output
- *
- * @param canopy
- */
+ /** Format the canopy for output */
public static String formatCanopy(Canopy canopy) {
return "C" + canopy.id + ": "
+ canopy.computeCentroid().asFormatString();
@@ -242,7 +228,7 @@
/**
* Decodes and returns a Canopy from the formattedString
- *
+ *
* @param formattedString a String prouced by formatCanopy
* @return a new Canopy
*/
@@ -261,7 +247,7 @@
/**
* Add a point to the canopy
- *
+ *
* @param point some point to add
*/
public void addPoint(Vector point) {
@@ -271,9 +257,8 @@
}
/**
- * Emit the point to the collector, keyed by the canopy's formatted
- * representation
- *
+ * Emit the point to the collector, keyed by the canopy's formatted representation
+ *
* @param point a point to emit.
*/
public void emitPoint(Vector point, OutputCollector<Text, Vector> collector)
@@ -291,19 +276,18 @@
}
-
/**
* Compute the centroid by averaging the pointTotals
- *
+ *
* @return a SparseVector (required by Mapper) which is the new centroid
*/
public Vector computeCentroid() {
- return pointTotal.divide(numPoints);
+ return pointTotal.divide(numPoints);
}
/**
* Return if the point is covered by this canopy
- *
+ *
* @param point a point
* @return if the point is covered
*/
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusteringJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusteringJob.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusteringJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusteringJob.java Fri Jul 10 09:35:19 2009
@@ -17,38 +17,35 @@
package org.apache.mahout.clustering.canopy;
-import org.apache.mahout.matrix.Vector;
-import org.apache.mahout.matrix.SparseVector;
-import org.apache.mahout.utils.CommandLineUtil;
-import org.apache.mahout.utils.SquaredEuclideanDistanceMeasure;
-import org.apache.commons.cli2.builder.GroupBuilder;
-import org.apache.commons.cli2.builder.ArgumentBuilder;
-import org.apache.commons.cli2.builder.DefaultOptionBuilder;
import org.apache.commons.cli2.CommandLine;
-import org.apache.commons.cli2.OptionException;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import org.apache.commons.cli2.commandline.Parser;
+import org.apache.mahout.matrix.SparseVector;
+import org.apache.mahout.matrix.Vector;
+import org.apache.mahout.utils.CommandLineUtil;
+import org.apache.mahout.utils.SquaredEuclideanDistanceMeasure;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
/**
- * Runs the {@link org.apache.mahout.clustering.canopy.CanopyDriver#runJob(String, String, String, double, double, Class)}
- * and then {@link org.apache.mahout.clustering.canopy.ClusterDriver#runJob(String, String, String, String, double, double, Class)}.
+ * Runs the {@link org.apache.mahout.clustering.canopy.CanopyDriver#runJob(String, String, String, double, double,
+ * Class)} and then {@link org.apache.mahout.clustering.canopy.ClusterDriver#runJob(String, String, String, String,
+ * double, double, Class)}.
*/
public final class CanopyClusteringJob {
private static final Logger log = LoggerFactory.getLogger(CanopyClusteringJob.class);
- /**
- * The default name of the canopies output sub-directory.
- */
+ /** The default name of the canopies output sub-directory. */
public static final String DEFAULT_CANOPIES_OUTPUT_DIRECTORY = "/canopies";
- /**
- * The default name of the directory used to output clusters.
- */
+ /** The default name of the directory used to output clusters. */
public static final String DEFAULT_CLUSTER_OUTPUT_DIRECTORY = ClusterDriver.DEFAULT_CLUSTER_OUTPUT_DIRECTORY;
private CanopyClusteringJob() {
@@ -63,36 +60,35 @@
GroupBuilder gbuilder = new GroupBuilder();
Option inputOpt = obuilder.withLongName("input").withRequired(true).withArgument(
- abuilder.withName("input").withMinimum(1).withMaximum(1).create()).
- withDescription("The Path for input Vectors. Must be a SequenceFile of Writable, Vector").withShortName("i").create();
+ abuilder.withName("input").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Path for input Vectors. Must be a SequenceFile of Writable, Vector").withShortName("i").create();
Option outputOpt = obuilder.withLongName("output").withRequired(true).withArgument(
- abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
- withDescription("The Path to put the output in").withShortName("o").create();
+ abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Path to put the output in").withShortName("o").create();
Option measureClassOpt = obuilder.withLongName("distance").withRequired(false).withArgument(
abuilder.withName("distance").withMinimum(1).withMaximum(1).create()).
withDescription("The Distance Measure to use. Default is SquaredEuclidean").withShortName("m").create();
Option vectorClassOpt = obuilder.withLongName("vectorClass").withRequired(false).withArgument(
- abuilder.withName("vectorClass").withMinimum(1).withMaximum(1).create()).
- withDescription("The Vector implementation class name. Default is SparseVector.class").withShortName("v").create();
+ abuilder.withName("vectorClass").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Vector implementation class name. Default is SparseVector.class").withShortName("v").create();
Option t1Opt = obuilder.withLongName("t1").withRequired(true).withArgument(
- abuilder.withName("t1").withMinimum(1).withMaximum(1).create()).
- withDescription("t1").withShortName("t1").create();
+ abuilder.withName("t1").withMinimum(1).withMaximum(1).create()).
+ withDescription("t1").withShortName("t1").create();
Option t2Opt = obuilder.withLongName("t2").withRequired(true).withArgument(
- abuilder.withName("t2").withMinimum(1).withMaximum(1).create()).
- withDescription("t2").withShortName("t2").create();
-
+ abuilder.withName("t2").withMinimum(1).withMaximum(1).create()).
+ withDescription("t2").withShortName("t2").create();
Option helpOpt = obuilder.withLongName("help").
- withDescription("Print out help").withShortName("h").create();
+ withDescription("Print out help").withShortName("h").create();
Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt)
- .withOption(measureClassOpt).withOption(vectorClassOpt)
- .withOption(t1Opt).withOption(t2Opt)
- .withOption(helpOpt).create();
+ .withOption(measureClassOpt).withOption(vectorClassOpt)
+ .withOption(t1Opt).withOption(t2Opt)
+ .withOption(helpOpt).create();
try {
@@ -113,8 +109,8 @@
}
Class<? extends Vector> vectorClass = cmdLine.hasOption(vectorClassOpt) == false ?
- SparseVector.class
- : (Class<? extends Vector>) Class.forName(cmdLine.getValue(vectorClassOpt).toString());
+ SparseVector.class
+ : (Class<? extends Vector>) Class.forName(cmdLine.getValue(vectorClassOpt).toString());
double t1 = Double.parseDouble(cmdLine.getValue(t1Opt).toString());
double t2 = Double.parseDouble(cmdLine.getValue(t2Opt).toString());
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java Fri Jul 10 09:35:19 2009
@@ -17,6 +17,14 @@
package org.apache.mahout.clustering.canopy;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
+import org.apache.commons.cli2.commandline.Parser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -26,18 +34,10 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
-import org.apache.mahout.matrix.Vector;
import org.apache.mahout.matrix.SparseVector;
+import org.apache.mahout.matrix.Vector;
import org.apache.mahout.utils.CommandLineUtil;
import org.apache.mahout.utils.SquaredEuclideanDistanceMeasure;
-import org.apache.commons.cli2.builder.GroupBuilder;
-import org.apache.commons.cli2.builder.ArgumentBuilder;
-import org.apache.commons.cli2.builder.DefaultOptionBuilder;
-import org.apache.commons.cli2.Option;
-import org.apache.commons.cli2.Group;
-import org.apache.commons.cli2.CommandLine;
-import org.apache.commons.cli2.OptionException;
-import org.apache.commons.cli2.commandline.Parser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,35 +56,35 @@
GroupBuilder gbuilder = new GroupBuilder();
Option inputOpt = obuilder.withLongName("input").withRequired(true).withArgument(
- abuilder.withName("input").withMinimum(1).withMaximum(1).create()).
- withDescription("The Path for input Vectors. Must be a SequenceFile of Writable, Vector").withShortName("i").create();
+ abuilder.withName("input").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Path for input Vectors. Must be a SequenceFile of Writable, Vector").withShortName("i").create();
Option outputOpt = obuilder.withLongName("output").withRequired(true).withArgument(
- abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
- withDescription("The Path to put the output in").withShortName("o").create();
+ abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Path to put the output in").withShortName("o").create();
Option measureClassOpt = obuilder.withLongName("distance").withRequired(false).withArgument(
abuilder.withName("distance").withMinimum(1).withMaximum(1).create()).
withDescription("The Distance Measure to use. Default is SquaredEuclidean").withShortName("m").create();
Option vectorClassOpt = obuilder.withLongName("vectorClass").withRequired(false).withArgument(
- abuilder.withName("vectorClass").withMinimum(1).withMaximum(1).create()).
- withDescription("The Vector implementation class name. Default is SparseVector.class").withShortName("v").create();
+ abuilder.withName("vectorClass").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Vector implementation class name. Default is SparseVector.class").withShortName("v").create();
Option t1Opt = obuilder.withLongName("t1").withRequired(true).withArgument(
- abuilder.withName("t1").withMinimum(1).withMaximum(1).create()).
- withDescription("t1").withShortName("t1").create();
+ abuilder.withName("t1").withMinimum(1).withMaximum(1).create()).
+ withDescription("t1").withShortName("t1").create();
Option t2Opt = obuilder.withLongName("t2").withRequired(true).withArgument(
- abuilder.withName("t2").withMinimum(1).withMaximum(1).create()).
- withDescription("t2").withShortName("t2").create();
+ abuilder.withName("t2").withMinimum(1).withMaximum(1).create()).
+ withDescription("t2").withShortName("t2").create();
Option helpOpt = obuilder.withLongName("help").
- withDescription("Print out help").withShortName("h").create();
+ withDescription("Print out help").withShortName("h").create();
Group group = gbuilder.withName("Options").withOption(inputOpt).withOption(outputOpt)
- .withOption(measureClassOpt).withOption(vectorClassOpt)
- .withOption(t1Opt).withOption(t2Opt)
- .withOption(helpOpt).create();
+ .withOption(measureClassOpt).withOption(vectorClassOpt)
+ .withOption(t1Opt).withOption(t2Opt)
+ .withOption(helpOpt).create();
try {
Parser parser = new Parser();
@@ -104,8 +104,8 @@
}
Class<? extends Vector> vectorClass = cmdLine.hasOption(vectorClassOpt) == false ?
- SparseVector.class
- : (Class<? extends Vector>) Class.forName(cmdLine.getValue(vectorClassOpt).toString());
+ SparseVector.class
+ : (Class<? extends Vector>) Class.forName(cmdLine.getValue(vectorClassOpt).toString());
double t1 = Double.parseDouble(cmdLine.getValue(t1Opt).toString());
double t2 = Double.parseDouble(cmdLine.getValue(t2Opt).toString());
@@ -114,7 +114,7 @@
log.error("Exception", e);
CommandLineUtil.printHelp(group);
- }
+ }
}
/**
@@ -126,17 +126,16 @@
* @param t1 the T1 distance threshold
* @param t2 the T2 distance threshold
* @param vectorClass the {@link Class} of Vector to use for the Map Output Key. Must be a concrete type
- *
* @see org.apache.mahout.matrix.SparseVector
* @see org.apache.mahout.matrix.DenseVector
*/
public static void runJob(String input, String output,
String measureClassName, double t1, double t2, Class<? extends Vector> vectorClass) throws IOException {
log.info("Input: " + input + " Out: " + output + " Measure: " + measureClassName + " t1: " + t1
- + " t2: " + t2 + " Vector Class: " + vectorClass.getSimpleName());
+ + " t2: " + t2 + " Vector Class: " + vectorClass.getSimpleName());
JobClient client = new JobClient();
JobConf conf = new JobConf(
- org.apache.mahout.clustering.canopy.CanopyDriver.class);
+ org.apache.mahout.clustering.canopy.CanopyDriver.class);
conf.set(Canopy.DISTANCE_MEASURE_KEY, measureClassName);
conf.set(Canopy.T1_KEY, String.valueOf(t1));
conf.set(Canopy.T2_KEY, String.valueOf(t2));
@@ -159,8 +158,9 @@
client.setConf(conf);
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
JobClient.runJob(conf);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java Fri Jul 10 09:35:19 2009
@@ -17,10 +17,6 @@
package org.apache.mahout.clustering.canopy;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapred.JobConf;
@@ -30,6 +26,10 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.mahout.matrix.Vector;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
public class CanopyMapper extends MapReduceBase implements
Mapper<WritableComparable<?>, Vector, Text, Vector> {
@@ -39,7 +39,7 @@
@Override
public void map(WritableComparable<?> key, Vector point,
- OutputCollector<Text, Vector> output, Reporter reporter) throws IOException {
+ OutputCollector<Text, Vector> output, Reporter reporter) throws IOException {
outputCollector = output;
Canopy.addPointToCanopies(point, canopies);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyReducer.java Fri Jul 10 09:35:19 2009
@@ -17,11 +17,6 @@
package org.apache.mahout.clustering.canopy;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
@@ -30,8 +25,13 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.mahout.matrix.Vector;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
public class CanopyReducer extends MapReduceBase implements
- Reducer<Text, Vector, Text, Canopy> {
+ Reducer<Text, Vector, Text, Canopy> {
private final List<Canopy> canopies = new ArrayList<Canopy>();
@@ -42,8 +42,9 @@
Vector point = values.next();
Canopy.addPointToCanopies(point, canopies);
}
- for (Canopy canopy : canopies)
+ for (Canopy canopy : canopies) {
output.collect(new Text(canopy.getIdentifier()), canopy);
+ }
}
@Override
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterDriver.java Fri Jul 10 09:35:19 2009
@@ -17,6 +17,14 @@
package org.apache.mahout.clustering.canopy;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
+import org.apache.commons.cli2.commandline.Parser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -27,18 +35,10 @@
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.lib.IdentityReducer;
-import org.apache.mahout.matrix.Vector;
import org.apache.mahout.matrix.SparseVector;
+import org.apache.mahout.matrix.Vector;
import org.apache.mahout.utils.CommandLineUtil;
import org.apache.mahout.utils.SquaredEuclideanDistanceMeasure;
-import org.apache.commons.cli2.builder.DefaultOptionBuilder;
-import org.apache.commons.cli2.builder.ArgumentBuilder;
-import org.apache.commons.cli2.builder.GroupBuilder;
-import org.apache.commons.cli2.Option;
-import org.apache.commons.cli2.Group;
-import org.apache.commons.cli2.CommandLine;
-import org.apache.commons.cli2.OptionException;
-import org.apache.commons.cli2.commandline.Parser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -60,39 +60,39 @@
GroupBuilder gbuilder = new GroupBuilder();
Option vectorClassOpt = obuilder.withLongName("vectorClass").withRequired(false).withArgument(
- abuilder.withName("vectorClass").withMinimum(1).withMaximum(1).create()).
- withDescription("The Vector implementation class name. Default is SparseVector.class")
- .withShortName("v").create();
+ abuilder.withName("vectorClass").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Vector implementation class name. Default is SparseVector.class")
+ .withShortName("v").create();
Option t1Opt = obuilder.withLongName("t1").withRequired(true).withArgument(
- abuilder.withName("t1").withMinimum(1).withMaximum(1).create()).
- withDescription("t1").withShortName("t1").create();
+ abuilder.withName("t1").withMinimum(1).withMaximum(1).create()).
+ withDescription("t1").withShortName("t1").create();
Option t2Opt = obuilder.withLongName("t2").withRequired(true).withArgument(
- abuilder.withName("t2").withMinimum(1).withMaximum(1).create()).
- withDescription("t2").withShortName("t2").create();
+ abuilder.withName("t2").withMinimum(1).withMaximum(1).create()).
+ withDescription("t2").withShortName("t2").create();
Option pointsOpt = obuilder.withLongName("points").withRequired(true).withArgument(
- abuilder.withName("points").withMinimum(1).withMaximum(1).create()).
- withDescription("The path containing the points").withShortName("p").create();
+ abuilder.withName("points").withMinimum(1).withMaximum(1).create()).
+ withDescription("The path containing the points").withShortName("p").create();
Option canopiesOpt = obuilder.withLongName("canopies").withRequired(true).withArgument(
- abuilder.withName("canopies").withMinimum(1).withMaximum(1).create()).
- withDescription("The location of the canopies, as a Path").withShortName("c").create();
+ abuilder.withName("canopies").withMinimum(1).withMaximum(1).create()).
+ withDescription("The location of the canopies, as a Path").withShortName("c").create();
Option measureClassOpt = obuilder.withLongName("distance").withRequired(false).withArgument(
- abuilder.withName("distance").withMinimum(1).withMaximum(1).create()).
- withDescription("The Distance Measure to use. Default is SquaredEuclidean").withShortName("m").create();
+ abuilder.withName("distance").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Distance Measure to use. Default is SquaredEuclidean").withShortName("m").create();
Option outputOpt = obuilder.withLongName("output").withRequired(true).withArgument(
- abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
- withDescription("The Path to put the output in").withShortName("o").create();
+ abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Path to put the output in").withShortName("o").create();
Option helpOpt = obuilder.withLongName("help").
- withDescription("Print out help").withShortName("h").create();
+ withDescription("Print out help").withShortName("h").create();
Group group = gbuilder.withName("Options").withOption(vectorClassOpt)
- .withOption(t1Opt).withOption(t2Opt)
- .withOption(pointsOpt).withOption(canopiesOpt).withOption(measureClassOpt).withOption(outputOpt)
- .withOption(helpOpt).create();
+ .withOption(t1Opt).withOption(t2Opt)
+ .withOption(pointsOpt).withOption(canopiesOpt).withOption(measureClassOpt).withOption(outputOpt)
+ .withOption(helpOpt).create();
try {
Parser parser = new Parser();
@@ -111,8 +111,8 @@
String canopies = cmdLine.getValue(canopiesOpt).toString();
String points = cmdLine.getValue(pointsOpt).toString();
Class<? extends Vector> vectorClass = cmdLine.hasOption(vectorClassOpt) == false ?
- SparseVector.class
- : (Class<? extends Vector>) Class.forName(cmdLine.getValue(vectorClassOpt).toString());
+ SparseVector.class
+ : (Class<? extends Vector>) Class.forName(cmdLine.getValue(vectorClassOpt).toString());
double t1 = Double.parseDouble(cmdLine.getValue(t1Opt).toString());
double t2 = Double.parseDouble(cmdLine.getValue(t2Opt).toString());
@@ -141,7 +141,7 @@
String measureClassName, double t1, double t2, Class<? extends Vector> vectorClass) throws IOException {
JobClient client = new JobClient();
JobConf conf = new JobConf(
- org.apache.mahout.clustering.canopy.ClusterDriver.class);
+ org.apache.mahout.clustering.canopy.ClusterDriver.class);
conf.set(Canopy.DISTANCE_MEASURE_KEY, measureClassName);
conf.set(Canopy.T1_KEY, String.valueOf(t1));
@@ -149,7 +149,7 @@
conf.set(Canopy.CANOPY_PATH_KEY, canopies);
conf.setInputFormat(SequenceFileInputFormat.class);
-
+
/*conf.setMapOutputKeyClass(Text.class);
conf.setMapOutputValueClass(SparseVector.class);*/
conf.setOutputKeyClass(Text.class);
@@ -165,8 +165,9 @@
client.setConf(conf);
FileSystem dfs = FileSystem.get(outPath.toUri(), conf);
- if (dfs.exists(outPath))
+ if (dfs.exists(outPath)) {
dfs.delete(outPath, true);
+ }
JobClient.runJob(conf);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java Fri Jul 10 09:35:19 2009
@@ -17,10 +17,6 @@
package org.apache.mahout.clustering.canopy;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
@@ -33,8 +29,12 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.mahout.matrix.Vector;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
public class ClusterMapper extends MapReduceBase implements
- Mapper<WritableComparable<?>, Vector, Text, Vector> {
+ Mapper<WritableComparable<?>, Vector, Text, Vector> {
private List<Canopy> canopies;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java Fri Jul 10 09:35:19 2009
@@ -16,19 +16,18 @@
*/
package org.apache.mahout.clustering.dirichlet;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-import java.lang.reflect.Type;
-
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
import org.apache.hadoop.io.Writable;
import org.apache.mahout.clustering.dirichlet.models.Model;
import org.apache.mahout.matrix.JsonVectorAdapter;
import org.apache.mahout.matrix.Vector;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.reflect.TypeToken;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.lang.reflect.Type;
public class DirichletCluster<O> implements Writable {
@@ -75,13 +74,7 @@
return gson.toJson(this, typeOfModel);
}
- /**
- * Reads a typed Model instance from the input stream
- *
- * @param in
- * @return
- * @throws IOException
- */
+ /** Reads a typed Model instance from the input stream */
@SuppressWarnings("unchecked")
public static Model<?> readModel(DataInput in) throws IOException {
String modelClassName = in.readUTF();
@@ -100,13 +93,7 @@
return model;
}
- /**
- * Writes a typed Model instance to the output stream
- *
- * @param out
- * @param model
- * @throws IOException
- */
+ /** Writes a typed Model instance to the output stream */
@SuppressWarnings("unchecked")
public static void writeModel(DataOutput out, Model<?> model) throws IOException {
out.writeUTF(model.getClass().getName());
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java Fri Jul 10 09:35:19 2009
@@ -17,58 +17,43 @@
* limitations under the License.
*/
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.mahout.clustering.dirichlet.models.Model;
import org.apache.mahout.clustering.dirichlet.models.ModelDistribution;
import org.apache.mahout.matrix.DenseVector;
import org.apache.mahout.matrix.TimesFunction;
import org.apache.mahout.matrix.Vector;
+import java.util.ArrayList;
+import java.util.List;
+
/**
- * Performs Bayesian mixture modeling.
- * <p/>
- * The idea is that we use a probabilistic mixture of a number of models that we use to explain some observed data.
- * The idea here is that each observed data point is assumed to have come from one of the models in the mixture, but
- * we don't know which. The way we deal with that is to use a so-called latent parameter which specifies which model
- * each data point came from.
- * <p/>
- * In addition, since this is a Bayesian clustering algorithm, we don't want to actually commit to any single explanation,
- * but rather to sample from the distribution of models and latent assignments of data points to models given the
- * observed data and the prior distributions of model parameters.
- * <p/>
- * This sampling process is initialized by taking models at random from the prior distribution for models.
- * <p/>
- * Then, we iteratively assign points to the different models using the mixture probabilities and the degree
- * of fit between the point and each model expressed as a probability that the point was generated by that model.
- * <p/>
- * After points are assigned, new parameters for each model are sampled from the posterior distribution for the model
- * parameters considering all of the observed data points that were assigned to the model. Models without any
- * data points are also sampled, but since they have no points assigned, the new samples are effectively taken from
- * the prior distribution for model parameters.
- * <p/>
- * The result is a number of samples that represent mixing probabilities, models and assignment of points to models.
- * If the total number of possible models is substantially larger than the number that ever have points assigned to
- * them, then this algorithm provides a (nearly) non-parametric clustering algorithm.
- * <p/>
- * These samples can give us interesting information that is lacking from a normal clustering that consists of a
- * single assignment of points to clusters. Firstly, by examining the number of models in each sample that actually
- * has any points assigned to it, we can get information about how many models (clusters) that the data support.
- * <p/>
- * Morevoer, by examining how often two points are assigned to the same model, we can get an approximate measure
- * of how likely these points are to be explained by the same model. Such soft membership information is difficult
- * to come by with conventional clustering methods.
- * <p/>
- * Finally, we can get an idea of the stability of how the data can be described. Typically, aspects of the data
- * with lots of data available wind up with stable descriptions while at the edges, there are aspects that are
- * phenomena that we can't really commit to a solid description, but it is still clear that the well supported
- * explanations are insufficient to explain these additional aspects.
- * <p/>
- * One thing that can be difficult about these samples is that we can't always assign a correlation between the models
- * in the different samples. Probably the best way to do this is to look for overlap in the assignments of data
- * observations to the different models.
- * <p/>
+ * Performs Bayesian mixture modeling. <p/> The idea is that we use a probabilistic mixture of a number of models that
+ * we use to explain some observed data. The idea here is that each observed data point is assumed to have come from one
+ * of the models in the mixture, but we don't know which. The way we deal with that is to use a so-called latent
+ * parameter which specifies which model each data point came from. <p/> In addition, since this is a Bayesian
+ * clustering algorithm, we don't want to actually commit to any single explanation, but rather to sample from the
+ * distribution of models and latent assignments of data points to models given the observed data and the prior
+ * distributions of model parameters. <p/> This sampling process is initialized by taking models at random from the
+ * prior distribution for models. <p/> Then, we iteratively assign points to the different models using the mixture
+ * probabilities and the degree of fit between the point and each model expressed as a probability that the point was
+ * generated by that model. <p/> After points are assigned, new parameters for each model are sampled from the posterior
+ * distribution for the model parameters considering all of the observed data points that were assigned to the model.
+ * Models without any data points are also sampled, but since they have no points assigned, the new samples are
+ * effectively taken from the prior distribution for model parameters. <p/> The result is a number of samples that
+ * represent mixing probabilities, models and assignment of points to models. If the total number of possible models is
+ * substantially larger than the number that ever have points assigned to them, then this algorithm provides a (nearly)
+ * non-parametric clustering algorithm. <p/> These samples can give us interesting information that is lacking from a
+ * normal clustering that consists of a single assignment of points to clusters. Firstly, by examining the number of
+ * models in each sample that actually has any points assigned to it, we can get information about how many models
+ * (clusters) that the data support. <p/> Morevoer, by examining how often two points are assigned to the same model, we
+ * can get an approximate measure of how likely these points are to be explained by the same model. Such soft
+ * membership information is difficult to come by with conventional clustering methods. <p/> Finally, we can get an idea
+ * of the stability of how the data can be described. Typically, aspects of the data with lots of data available wind
+ * up with stable descriptions while at the edges, there are aspects that are phenomena that we can't really commit to a
+ * solid description, but it is still clear that the well supported explanations are insufficient to explain these
+ * additional aspects. <p/> One thing that can be difficult about these samples is that we can't always assign a
+ * correlation between the models in the different samples. Probably the best way to do this is to look for overlap in
+ * the assignments of data observations to the different models. <p/>
* <pre>
* \theta_i ~ prior()
* \lambda_i ~ Dirichlet(\alpha_0)
@@ -98,16 +83,16 @@
/**
* Create a new instance on the sample data with the given additional parameters
*
- * @param sampleData the observed data to be clustered
- * @param modelFactory the ModelDistribution to use
- * @param alpha_0 the double value for the beta distributions
- * @param numClusters the int number of clusters
- * @param thin the int thinning interval, used to report every n iterations
- * @param burnin the int burnin interval, used to suppress early iterations
+ * @param sampleData the observed data to be clustered
+ * @param modelFactory the ModelDistribution to use
+ * @param alpha_0 the double value for the beta distributions
+ * @param numClusters the int number of clusters
+ * @param thin the int thinning interval, used to report every n iterations
+ * @param burnin the int burnin interval, used to suppress early iterations
*/
public DirichletClusterer(List<O> sampleData,
- ModelDistribution<O> modelFactory, double alpha_0,
- int numClusters, int thin, int burnin) {
+ ModelDistribution<O> modelFactory, double alpha_0,
+ int numClusters, int thin, int burnin) {
this.sampleData = sampleData;
this.modelFactory = modelFactory;
this.thin = thin;
@@ -118,22 +103,22 @@
}
/**
- * Iterate over the sample data, obtaining cluster samples periodically
- * and returning them.
- *
+ * Iterate over the sample data, obtaining cluster samples periodically and returning them.
+ *
* @param numIterations the int number of iterations to perform
* @return a List<List<Model<Observation>>> of the observed models
*/
public List<Model<O>[]> cluster(int numIterations) {
- for (int iteration = 0; iteration < numIterations; iteration++)
+ for (int iteration = 0; iteration < numIterations; iteration++) {
iterate(iteration, state);
+ }
return clusterSamples;
}
/**
- * Perform one iteration of the clustering process, iterating over the samples to build
- * a new array of models, then updating the state for the next iteration
- *
+ * Perform one iteration of the clustering process, iterating over the samples to build a new array of models, then
+ * updating the state for the next iteration
+ *
* @param state the DirichletState<Observation> of this iteration
*/
private void iterate(int iteration, DirichletState<O> state) {
@@ -154,29 +139,31 @@
}
// periodically add models to the cluster samples after the burn-in period
- if ((iteration >= burnin) && (iteration % thin == 0))
+ if ((iteration >= burnin) && (iteration % thin == 0)) {
clusterSamples.add(newModels);
+ }
// update the state from the new models
state.update(newModels);
}
/**
- * Compute a normalized vector of probabilities that x is described
- * by each model using the mixture and the model pdfs
- *
+ * Compute a normalized vector of probabilities that x is described by each model using the mixture and the model
+ * pdfs
+ *
* @param state the DirichletState<Observation> of this iteration
- * @param x an Observation
+ * @param x an Observation
* @return the Vector of probabilities
*/
private Vector normalizedProbabilities(DirichletState<O> state,
- O x) {
+ O x) {
Vector pi = new DenseVector(numClusters);
double max = 0;
for (int k = 0; k < numClusters; k++) {
double p = state.adjustedProbability(x, k);
pi.set(k, p);
- if (max < p)
+ if (max < p) {
max = p;
+ }
}
// normalize the probabilities by largest observed value
pi.assign(new TimesFunction(), 1.0 / max);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCombiner.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCombiner.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCombiner.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCombiner.java Fri Jul 10 09:35:19 2009
@@ -17,21 +17,21 @@
package org.apache.mahout.clustering.dirichlet;
-import java.io.IOException;
-import java.util.Iterator;
-
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
+import java.io.IOException;
+import java.util.Iterator;
+
public class DirichletCombiner extends MapReduceBase implements
Reducer<Text, Text, Text, Text> {
@Override
public void reduce(Text key, Iterator<Text> values,
- OutputCollector<Text, Text> output, Reporter reporter) throws IOException {
+ OutputCollector<Text, Text> output, Reporter reporter) throws IOException {
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java Fri Jul 10 09:35:19 2009
@@ -17,8 +17,6 @@
package org.apache.mahout.clustering.dirichlet;
-import java.io.IOException;
-
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.SequenceFile;
@@ -36,6 +34,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+
public class DirichletDriver {
public static final String STATE_IN_KEY = "org.apache.mahout.clustering.dirichlet.stateIn";
@@ -68,20 +68,16 @@
/**
* Run the job using supplied arguments
*
- * @param input the directory pathname for input points
- * @param output the directory pathname for output points
- * @param modelFactory the String ModelDistribution class name to use
- * @param numClusters the number of models
- * @param maxIterations the maximum number of iterations
- * @param alpha_0 the alpha_0 value for the DirichletDistribution
- * @param numReducers the number of Reducers desired
- * @throws ClassNotFoundException
- * @throws IllegalAccessException
- * @throws InstantiationException
- * @throws IOException
+ * @param input the directory pathname for input points
+ * @param output the directory pathname for output points
+ * @param modelFactory the String ModelDistribution class name to use
+ * @param numClusters the number of models
+ * @param maxIterations the maximum number of iterations
+ * @param alpha_0 the alpha_0 value for the DirichletDistribution
+ * @param numReducers the number of Reducers desired
*/
public static void runJob(String input, String output, String modelFactory,
- int numClusters, int maxIterations, double alpha_0, int numReducers)
+ int numClusters, int maxIterations, double alpha_0, int numReducers)
throws ClassNotFoundException, InstantiationException,
IllegalAccessException, IOException {
@@ -100,7 +96,7 @@
}
private static void writeInitialState(String output, String stateIn,
- String modelFactory, int numModels, double alpha_0)
+ String modelFactory, int numModels, double alpha_0)
throws ClassNotFoundException, InstantiationException,
IllegalAccessException, IOException {
DirichletState<Vector> state = createState(modelFactory, numModels, alpha_0);
@@ -119,7 +115,7 @@
@SuppressWarnings("unchecked")
public static DirichletState<Vector> createState(String modelFactory,
- int numModels, double alpha_0) throws ClassNotFoundException,
+ int numModels, double alpha_0) throws ClassNotFoundException,
InstantiationException, IllegalAccessException {
ClassLoader ccl = Thread.currentThread().getContextClassLoader();
Class<?> cl = ccl.loadClass(modelFactory);
@@ -132,17 +128,17 @@
/**
* Run the job using supplied arguments
*
- * @param input the directory pathname for input points
- * @param stateIn the directory pathname for input state
- * @param stateOut the directory pathname for output state
- * @param modelFactory the class name of the model factory class
- * @param numClusters the number of clusters
- * @param alpha_0 alpha_0
- * @param numReducers the number of Reducers desired
+ * @param input the directory pathname for input points
+ * @param stateIn the directory pathname for input state
+ * @param stateOut the directory pathname for output state
+ * @param modelFactory the class name of the model factory class
+ * @param numClusters the number of clusters
+ * @param alpha_0 alpha_0
+ * @param numReducers the number of Reducers desired
*/
public static void runIteration(String input, String stateIn,
- String stateOut, String modelFactory, int numClusters, double alpha_0,
- int numReducers) {
+ String stateOut, String modelFactory, int numClusters, double alpha_0,
+ int numReducers) {
JobClient client = new JobClient();
JobConf conf = new JobConf(DirichletDriver.class);
@@ -176,9 +172,9 @@
/**
* Run the job using supplied arguments
*
- * @param input the directory pathname for input points
- * @param stateIn the directory pathname for input state
- * @param output the directory pathname for output points
+ * @param input the directory pathname for input points
+ * @param stateIn the directory pathname for input state
+ * @param output the directory pathname for output points
*/
public static void runClustering(String input, String stateIn, String output) {
JobClient client = new JobClient();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletJob.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletJob.java Fri Jul 10 09:35:19 2009
@@ -17,12 +17,12 @@
package org.apache.mahout.clustering.dirichlet;
-import java.io.IOException;
-
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
+import java.io.IOException;
+
public class DirichletJob {
private DirichletJob() {
@@ -40,21 +40,17 @@
}
/**
- * Run the job using supplied arguments, deleting the output directory if it
- * exists beforehand
- *
- * @param input the directory pathname for input points
- * @param output the directory pathname for output points
- * @param modelFactory the ModelDistribution class name
- * @param numModels the number of Models
+ * Run the job using supplied arguments, deleting the output directory if it exists beforehand
+ *
+ * @param input the directory pathname for input points
+ * @param output the directory pathname for output points
+ * @param modelFactory the ModelDistribution class name
+ * @param numModels the number of Models
* @param maxIterations the maximum number of iterations
- * @param alpha_0 the alpha0 value for the DirichletDistribution
- * @throws IllegalAccessException
- * @throws InstantiationException
- * @throws ClassNotFoundException
+ * @param alpha_0 the alpha0 value for the DirichletDistribution
*/
public static void runJob(String input, String output, String modelFactory,
- int numModels, int maxIterations, double alpha_0)
+ int numModels, int maxIterations, double alpha_0)
throws IOException, ClassNotFoundException, InstantiationException,
IllegalAccessException {
// delete the output directory
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java Fri Jul 10 09:35:19 2009
@@ -17,8 +17,6 @@
package org.apache.mahout.clustering.dirichlet;
-import java.io.IOException;
-
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -34,6 +32,8 @@
import org.apache.mahout.matrix.TimesFunction;
import org.apache.mahout.matrix.Vector;
+import java.io.IOException;
+
public class DirichletMapper extends MapReduceBase implements
Mapper<WritableComparable<?>, Vector, Text, Vector> {
@@ -41,7 +41,7 @@
@Override
public void map(WritableComparable<?> key, Vector v,
- OutputCollector<Text, Vector> output, Reporter reporter) throws IOException {
+ OutputCollector<Text, Vector> output, Reporter reporter) throws IOException {
// compute a normalized vector of probabilities that v is described by each model
Vector pi = normalizedProbabilities(state, v);
// then pick one model by sampling a Multinomial distribution based upon them
@@ -97,11 +97,11 @@
}
/**
- * Compute a normalized vector of probabilities that v is described
- * by each model using the mixture and the model pdfs
- *
+ * Compute a normalized vector of probabilities that v is described by each model using the mixture and the model
+ * pdfs
+ *
* @param state the DirichletState<Vector> of this iteration
- * @param v an Vector
+ * @param v an Vector
* @return the Vector of probabilities
*/
private static Vector normalizedProbabilities(DirichletState<Vector> state, Vector v) {
@@ -110,8 +110,9 @@
for (int k = 0; k < state.numClusters; k++) {
double p = state.adjustedProbability(v, k);
pi.set(k, p);
- if (max < p)
+ if (max < p) {
max = p;
+ }
}
// normalize the probabilities by largest observed value
pi.assign(new TimesFunction(), 1.0 / max);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java Fri Jul 10 09:35:19 2009
@@ -17,9 +17,6 @@
package org.apache.mahout.clustering.dirichlet;
-import java.io.IOException;
-import java.util.Iterator;
-
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
@@ -29,6 +26,9 @@
import org.apache.mahout.clustering.dirichlet.models.Model;
import org.apache.mahout.matrix.Vector;
+import java.io.IOException;
+import java.util.Iterator;
+
public class DirichletReducer extends MapReduceBase implements
Reducer<Text, Vector, Text, DirichletCluster<Vector>> {
@@ -38,7 +38,7 @@
@Override
public void reduce(Text key, Iterator<Vector> values,
- OutputCollector<Text, DirichletCluster<Vector>> output, Reporter reporter)
+ OutputCollector<Text, DirichletCluster<Vector>> output, Reporter reporter)
throws IOException {
int k = Integer.parseInt(key.toString());
Model<Vector> model = newModels[k];
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java Fri Jul 10 09:35:19 2009
@@ -17,14 +17,14 @@
* limitations under the License.
*/
-import java.util.ArrayList;
-import java.util.List;
-
import org.apache.mahout.clustering.dirichlet.models.Model;
import org.apache.mahout.clustering.dirichlet.models.ModelDistribution;
import org.apache.mahout.matrix.DenseVector;
import org.apache.mahout.matrix.Vector;
+import java.util.ArrayList;
+import java.util.List;
+
public class DirichletState<O> {
public int numClusters; // the number of clusters
@@ -39,15 +39,16 @@
@SuppressWarnings("unchecked")
public DirichletState(ModelDistribution<O> modelFactory,
- int numClusters, double alpha_0, int thin, int burnin) {
+ int numClusters, double alpha_0, int thin, int burnin) {
this.numClusters = numClusters;
this.modelFactory = modelFactory;
// initialize totalCounts
offset = alpha_0 / numClusters;
// sample initial prior models
clusters = new ArrayList<DirichletCluster<O>>();
- for (Model<?> m : modelFactory.sampleFromPrior(numClusters))
+ for (Model<?> m : modelFactory.sampleFromPrior(numClusters)) {
clusters.add(new DirichletCluster(m, offset));
+ }
// sample the mixture parameters from a Dirichlet distribution on the totalCounts
mixture = UncommonDistributions.rDirichlet(totalCounts());
}
@@ -57,14 +58,15 @@
public Vector totalCounts() {
Vector result = new DenseVector(numClusters);
- for (int i = 0; i < numClusters; i++)
+ for (int i = 0; i < numClusters; i++) {
result.set(i, clusters.get(i).totalCount);
+ }
return result;
}
/**
* Update the receiver with the new models
- *
+ *
* @param newModels a Model<Observation>[] of new models
*/
public void update(Model<O>[] newModels) {
@@ -79,6 +81,7 @@
/**
* return the adjusted probability that x is described by the kth model
+ *
* @param x an Observation
* @param k an int index of a model
* @return the double probability
@@ -92,8 +95,9 @@
@SuppressWarnings("unchecked")
public Model<O>[] getModels() {
Model<O>[] result = new Model[numClusters];
- for (int i = 0; i < numClusters; i++)
+ for (int i = 0; i < numClusters; i++) {
result[i] = clusters.get(i).model;
+ }
return result;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonDirichletStateAdapter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonDirichletStateAdapter.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonDirichletStateAdapter.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonDirichletStateAdapter.java Fri Jul 10 09:35:19 2009
@@ -16,14 +16,6 @@
*/
package org.apache.mahout.clustering.dirichlet;
-import java.lang.reflect.Type;
-import java.util.List;
-
-import org.apache.mahout.clustering.dirichlet.models.Model;
-import org.apache.mahout.clustering.dirichlet.models.ModelDistribution;
-import org.apache.mahout.matrix.JsonVectorAdapter;
-import org.apache.mahout.matrix.Vector;
-
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
@@ -35,6 +27,13 @@
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import com.google.gson.reflect.TypeToken;
+import org.apache.mahout.clustering.dirichlet.models.Model;
+import org.apache.mahout.clustering.dirichlet.models.ModelDistribution;
+import org.apache.mahout.matrix.JsonVectorAdapter;
+import org.apache.mahout.matrix.Vector;
+
+import java.lang.reflect.Type;
+import java.util.List;
@SuppressWarnings("unchecked")
public class JsonDirichletStateAdapter implements
@@ -48,7 +47,7 @@
@Override
public JsonElement serialize(DirichletState<?> src, Type typeOfSrc,
- JsonSerializationContext context) {
+ JsonSerializationContext context) {
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
builder.registerTypeAdapter(Model.class, new JsonModelAdapter());
@@ -69,7 +68,7 @@
@Override
public DirichletState<?> deserialize(JsonElement json, Type typeOfT,
- JsonDeserializationContext context) throws JsonParseException {
+ JsonDeserializationContext context) throws JsonParseException {
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
builder.registerTypeAdapter(Model.class, new JsonModelAdapter());
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelAdapter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelAdapter.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelAdapter.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelAdapter.java Fri Jul 10 09:35:19 2009
@@ -16,14 +16,6 @@
*/
package org.apache.mahout.clustering.dirichlet;
-import java.lang.reflect.Type;
-
-import org.apache.mahout.clustering.dirichlet.models.Model;
-import org.apache.mahout.matrix.JsonVectorAdapter;
-import org.apache.mahout.matrix.Vector;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonDeserializationContext;
@@ -34,6 +26,13 @@
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
+import org.apache.mahout.clustering.dirichlet.models.Model;
+import org.apache.mahout.matrix.JsonVectorAdapter;
+import org.apache.mahout.matrix.Vector;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Type;
public class JsonModelAdapter implements JsonSerializer<Model<?>>,
JsonDeserializer<Model<?>> {
@@ -42,7 +41,7 @@
@Override
public JsonElement serialize(Model<?> src, Type typeOfSrc,
- JsonSerializationContext context) {
+ JsonSerializationContext context) {
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
Gson gson = builder.create();
@@ -54,7 +53,7 @@
@Override
public Model<?> deserialize(JsonElement json, Type typeOfT,
- JsonDeserializationContext context) throws JsonParseException {
+ JsonDeserializationContext context) throws JsonParseException {
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Vector.class, new JsonVectorAdapter());
Gson gson = builder.create();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelDistributionAdapter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelDistributionAdapter.java?rev=792856&r1=792855&r2=792856&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelDistributionAdapter.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/JsonModelDistributionAdapter.java Fri Jul 10 09:35:19 2009
@@ -16,12 +16,6 @@
*/
package org.apache.mahout.clustering.dirichlet;
-import java.lang.reflect.Type;
-
-import org.apache.mahout.clustering.dirichlet.models.ModelDistribution;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
@@ -29,6 +23,11 @@
import com.google.gson.JsonPrimitive;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
+import org.apache.mahout.clustering.dirichlet.models.ModelDistribution;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Type;
public class JsonModelDistributionAdapter implements
JsonSerializer<ModelDistribution<?>>, JsonDeserializer<ModelDistribution<?>> {
@@ -37,13 +36,13 @@
@Override
public JsonElement serialize(ModelDistribution<?> src, Type typeOfSrc,
- JsonSerializationContext context) {
+ JsonSerializationContext context) {
return new JsonPrimitive(src.getClass().getName());
}
@Override
public ModelDistribution<?> deserialize(JsonElement json, Type typeOfT,
- JsonDeserializationContext context) throws JsonParseException {
+ JsonDeserializationContext context) throws JsonParseException {
ClassLoader ccl = Thread.currentThread().getContextClassLoader();
Class<?> cl;
try {