You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2010/05/28 22:03:40 UTC
svn commit: r949271 [2/5] - in /mahout/trunk: buildtools/src/main/resources/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/si...
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/bayes/BayesClassifierDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/bayes/BayesClassifierDriver.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/bayes/BayesClassifierDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/mapreduce/bayes/BayesClassifierDriver.java Fri May 28 20:03:36 2010
@@ -46,7 +46,7 @@ public final class BayesClassifierDriver
private static final Logger log = LoggerFactory.getLogger(BayesClassifierDriver.class);
- private BayesClassifierDriver() {}
+ private BayesClassifierDriver() { }
/**
* Run the job
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/ClusterBase.java Fri May 28 20:03:36 2010
@@ -108,8 +108,7 @@ public abstract class ClusterBase implem
@Override
public String asJsonString() {
- Type vectorType = new TypeToken<Vector>() {
- }.getType();
+ Type vectorType = new TypeToken<Vector>() {}.getType();
GsonBuilder gBuilder = new GsonBuilder();
gBuilder.registerTypeAdapter(vectorType, new JsonVectorAdapter());
Gson gson = gBuilder.create();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/WeightedVectorWritable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/WeightedVectorWritable.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/WeightedVectorWritable.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/WeightedVectorWritable.java Fri May 28 20:03:36 2010
@@ -30,6 +30,14 @@ public class WeightedVectorWritable impl
private VectorWritable vector;
+ public WeightedVectorWritable() {
+ }
+
+ public WeightedVectorWritable(double weight, VectorWritable vector) {
+ this.weight = weight;
+ this.vector = vector;
+ }
+
/**
* @return the weight
*/
@@ -44,16 +52,6 @@ public class WeightedVectorWritable impl
return vector;
}
- public WeightedVectorWritable(double weight, VectorWritable vector) {
- super();
- this.weight = weight;
- this.vector = vector;
- }
-
- public WeightedVectorWritable() {
- super();
- }
-
@Override
public void readFields(DataInput in) throws IOException {
weight = in.readDouble();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java Fri May 28 20:03:36 2010
@@ -65,8 +65,9 @@ public final class CanopyDriver {
Option overwriteOutput = DefaultOptionCreator.overwriteOption().create();
Option clusteringOpt = DefaultOptionCreator.clusteringOption().create();
- Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(outputOpt).withOption(overwriteOutput).withOption(
- measureClassOpt).withOption(t1Opt).withOption(t2Opt).withOption(clusteringOpt).withOption(helpOpt).create();
+ Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(outputOpt)
+ .withOption(overwriteOutput).withOption(measureClassOpt).withOption(t1Opt).withOption(t2Opt)
+ .withOption(clusteringOpt).withOption(helpOpt).create();
try {
Parser parser = new Parser();
@@ -112,9 +113,14 @@ public final class CanopyDriver {
* @param runClustering
* true if points are to be clustered after clusters are determined
*/
- public static void runJob(Path input, Path output, String measureClassName, double t1, double t2, boolean runClustering)
- throws IOException {
- log.info("Input: {} Out: {} " + "Measure: {} t1: {} t2: {}", new Object[] { input, output, measureClassName, t1, t2 });
+ public static void runJob(Path input,
+ Path output,
+ String measureClassName,
+ double t1,
+ double t2,
+ boolean runClustering) throws IOException {
+ log.info("Input: {} Out: {} " + "Measure: {} t1: {} t2: {}",
+ new Object[] {input, output, measureClassName, t1, t2});
Configurable client = new JobClient();
JobConf conf = new JobConf(CanopyDriver.class);
conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, measureClassName);
@@ -163,8 +169,12 @@ public final class CanopyDriver {
* @param t2
* the T2 distance threshold
*/
- public static void runClustering(Path points, Path canopies, Path output, String measureClassName, double t1, double t2)
- throws IOException {
+ public static void runClustering(Path points,
+ Path canopies,
+ Path output,
+ String measureClassName,
+ double t1,
+ double t2) throws IOException {
Configurable client = new JobClient();
JobConf conf = new JobConf(CanopyDriver.class);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java Fri May 28 20:03:36 2010
@@ -44,8 +44,10 @@ public class ClusterMapper extends MapRe
private final List<Canopy> canopies = new ArrayList<Canopy>();
@Override
- public void map(WritableComparable<?> key, VectorWritable point, OutputCollector<IntWritable, WeightedVectorWritable> output,
- Reporter reporter) throws IOException {
+ public void map(WritableComparable<?> key,
+ VectorWritable point,
+ OutputCollector<IntWritable, WeightedVectorWritable> output,
+ Reporter reporter) throws IOException {
canopyClusterer.emitPointToClosestCanopy(point.get(), canopies, output, reporter);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java Fri May 28 20:03:36 2010
@@ -32,36 +32,23 @@ import com.google.gson.reflect.TypeToken
public class DirichletCluster<O> implements Writable, Cluster {
- @Override
- public void readFields(DataInput in) throws IOException {
- this.totalCount = in.readDouble();
- this.model = readModel(in);
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- out.writeDouble(totalCount);
- writeModel(out, model);
- }
+ private static final Type CLUSTER_TYPE = new TypeToken<DirichletCluster<Vector>>() {}.getType();
private Model<O> model; // the model for this iteration
private double totalCount; // total count of observations for the model
public DirichletCluster(Model<O> model, double totalCount) {
- super();
this.model = model;
this.totalCount = totalCount;
}
public DirichletCluster(Model<O> model) {
- super();
this.model = model;
this.totalCount = 0.0;
}
public DirichletCluster() {
- super();
}
public Model<O> getModel() {
@@ -77,8 +64,17 @@ public class DirichletCluster<O> impleme
return totalCount;
}
- private static final Type clusterType = new TypeToken<DirichletCluster<Vector>>() {
- }.getType();
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ this.totalCount = in.readDouble();
+ this.model = readModel(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeDouble(totalCount);
+ writeModel(out, model);
+ }
/** Reads a typed Model instance from the input stream */
public static <O> Model<O> readModel(DataInput in) throws IOException {
@@ -113,7 +109,7 @@ public class DirichletCluster<O> impleme
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Model.class, new JsonModelAdapter());
Gson gson = builder.create();
- return gson.toJson(this, clusterType);
+ return gson.toJson(this, CLUSTER_TYPE);
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java Fri May 28 20:03:36 2010
@@ -42,16 +42,14 @@ import org.apache.mahout.math.VectorWrit
public class DirichletClusterMapper extends MapReduceBase implements
Mapper<WritableComparable<?>, VectorWritable, IntWritable, WeightedVectorWritable> {
- private OutputCollector<IntWritable, VectorWritable> output;
-
private List<DirichletCluster> clusters;
-
private DirichletClusterer clusterer;
- @SuppressWarnings("unchecked")
@Override
- public void map(WritableComparable<?> key, VectorWritable vector, OutputCollector<IntWritable, WeightedVectorWritable> output,
- Reporter reporter) throws IOException {
+ public void map(WritableComparable<?> key,
+ VectorWritable vector,
+ OutputCollector<IntWritable, WeightedVectorWritable> output,
+ Reporter reporter) throws IOException {
clusterer.emitPointToClusters(vector, clusters, output);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterer.java Fri May 28 20:03:36 2010
@@ -114,7 +114,7 @@ public class DirichletClusterer<O> {
* the observed data to be clustered
* @param modelFactory
* the ModelDistribution to use
- * @param alpha_0
+ * @param alpha0
* the double value for the beta distributions
* @param numClusters
* the int number of clusters
@@ -123,14 +123,18 @@ public class DirichletClusterer<O> {
* @param burnin
* the int burnin interval, used to suppress early iterations
*/
- public DirichletClusterer(List<O> sampleData, ModelDistribution<O> modelFactory, double alpha_0, int numClusters, int thin,
- int burnin) {
+ public DirichletClusterer(List<O> sampleData,
+ ModelDistribution<O> modelFactory,
+ double alpha0,
+ int numClusters,
+ int thin,
+ int burnin) {
this.sampleData = sampleData;
this.modelFactory = modelFactory;
this.thin = thin;
this.burnin = burnin;
this.numClusters = numClusters;
- state = new DirichletState<O>(modelFactory, numClusters, alpha_0);
+ state = new DirichletState<O>(modelFactory, numClusters, alpha0);
}
/**
@@ -139,7 +143,6 @@ public class DirichletClusterer<O> {
* @param threshold
*/
public DirichletClusterer(boolean emitMostLikely, double threshold) {
- super();
this.sampleData = null;
this.modelFactory = null;
this.thin = 0;
@@ -274,7 +277,7 @@ public class DirichletClusterer<O> {
* the observed data to be clustered
* @param modelFactory
* the ModelDistribution to use
- * @param alpha_0
+ * @param alpha0
* the double value for the beta distributions
* @param numClusters
* the int number of clusters
@@ -285,9 +288,15 @@ public class DirichletClusterer<O> {
* @param numIterations
* number of iterations to be performed
*/
- public static List<Model<Vector>[]> clusterPoints(List<Vector> points, ModelDistribution<Vector> modelFactory, double alpha_0,
- int numClusters, int thin, int burnin, int numIterations) {
- DirichletClusterer<Vector> clusterer = new DirichletClusterer<Vector>(points, modelFactory, alpha_0, numClusters, thin, burnin);
+ public static List<Model<Vector>[]> clusterPoints(List<Vector> points,
+ ModelDistribution<Vector> modelFactory,
+ double alpha0,
+ int numClusters,
+ int thin,
+ int burnin,
+ int numIterations) {
+ DirichletClusterer<Vector> clusterer =
+ new DirichletClusterer<Vector>(points, modelFactory, alpha0, numClusters, thin, burnin);
return clusterer.cluster(numIterations);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java Fri May 28 20:03:36 2010
@@ -91,9 +91,11 @@ public final class DirichletDriver {
Option emitMostLikelyOpt = DefaultOptionCreator.emitMostLikelyOption().create();
Option thresholdOpt = DefaultOptionCreator.thresholdOption().create();
- Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(outputOpt).withOption(overwriteOutput).withOption(
- modelDistOpt).withOption(prototypeOpt).withOption(maxIterOpt).withOption(alphaOpt).withOption(kOpt).withOption(helpOpt)
- .withOption(numRedOpt).withOption(clusteringOpt).withOption(emitMostLikelyOpt).withOption(thresholdOpt).create();
+ Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(outputOpt)
+ .withOption(overwriteOutput).withOption(modelDistOpt).withOption(prototypeOpt)
+ .withOption(maxIterOpt).withOption(alphaOpt).withOption(kOpt).withOption(helpOpt)
+ .withOption(numRedOpt).withOption(clusteringOpt).withOption(emitMostLikelyOpt)
+ .withOption(thresholdOpt).create();
try {
Parser parser = new Parser();
@@ -117,9 +119,9 @@ public final class DirichletDriver {
int maxIterations = Integer.parseInt(cmdLine.getValue(maxIterOpt).toString());
boolean emitMostLikely = Boolean.parseBoolean(cmdLine.getValue(emitMostLikelyOpt).toString());
double threshold = Double.parseDouble(cmdLine.getValue(thresholdOpt).toString());
- double alpha_0 = Double.parseDouble(cmdLine.getValue(alphaOpt).toString());
+ double alpha0 = Double.parseDouble(cmdLine.getValue(alphaOpt).toString());
- runJob(input, output, modelFactory, modelPrototype, numModels, maxIterations, alpha_0, numReducers, cmdLine
+ runJob(input, output, modelFactory, modelPrototype, numModels, maxIterations, alpha0, numReducers, cmdLine
.hasOption(clusteringOpt), emitMostLikely, threshold);
} catch (OptionException e) {
log.error("Exception parsing command line: ", e);
@@ -142,7 +144,7 @@ public final class DirichletDriver {
* the number of models
* @param maxIterations
* the maximum number of iterations
- * @param alpha_0
+ * @param alpha0
* the alpha_0 value for the DirichletDistribution
* @param numReducers
* the number of Reducers desired
@@ -153,22 +155,39 @@ public final class DirichletDriver {
* @param threshold
* a double threshold value emits all clusters having greater pdf (emitMostLikely = false)
*/
- public static void runJob(Path input, Path output, String modelFactory, String modelPrototype, int numClusters,
- int maxIterations, double alpha_0, int numReducers, boolean runClustering, boolean emitMostLikely, double threshold)
- throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException, SecurityException,
- NoSuchMethodException, InvocationTargetException {
+ public static void runJob(Path input,
+ Path output,
+ String modelFactory,
+ String modelPrototype,
+ int numClusters,
+ int maxIterations,
+ double alpha0,
+ int numReducers,
+ boolean runClustering,
+ boolean emitMostLikely,
+ double threshold)
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException,
+ SecurityException, NoSuchMethodException, InvocationTargetException {
Path clustersIn = new Path(output, Cluster.INITIAL_CLUSTERS_DIR);
int protoSize = readPrototypeSize(input);
- writeInitialState(output, clustersIn, modelFactory, modelPrototype, protoSize, numClusters, alpha_0);
+ writeInitialState(output, clustersIn, modelFactory, modelPrototype, protoSize, numClusters, alpha0);
for (int iteration = 1; iteration <= maxIterations; iteration++) {
log.info("Iteration {}", iteration);
// point the output to a new directory per iteration
Path clustersOut = new Path(output, Cluster.CLUSTERS_DIR + iteration);
- runIteration(input, clustersIn, clustersOut, modelFactory, modelPrototype, protoSize, numClusters, alpha_0, numReducers);
+ runIteration(input,
+ clustersIn,
+ clustersOut,
+ modelFactory,
+ modelPrototype,
+ protoSize,
+ numClusters,
+ alpha0,
+ numReducers);
// now point the input to the old output directory
clustersIn = clustersOut;
}
@@ -196,11 +215,17 @@ public final class DirichletDriver {
return protoSize;
}
- private static void writeInitialState(Path output, Path stateIn, String modelFactory, String modelPrototype, int prototypeSize,
- int numModels, double alpha_0) throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException,
- SecurityException, NoSuchMethodException, InvocationTargetException {
+ private static void writeInitialState(Path output,
+ Path stateIn,
+ String modelFactory,
+ String modelPrototype,
+ int prototypeSize,
+ int numModels,
+ double alpha0)
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException, IOException,
+ SecurityException, NoSuchMethodException, InvocationTargetException {
- DirichletState<VectorWritable> state = createState(modelFactory, modelPrototype, prototypeSize, numModels, alpha_0);
+ DirichletState<VectorWritable> state = createState(modelFactory, modelPrototype, prototypeSize, numModels, alpha0);
JobConf job = new JobConf(DirichletDriver.class);
FileSystem fs = FileSystem.get(output.toUri(), job);
for (int i = 0; i < numModels; i++) {
@@ -223,13 +248,17 @@ public final class DirichletDriver {
* an int number of dimensions of the model prototype vector
* @param numModels
* an int number of models to be created
- * @param alpha_0
+ * @param alpha0
* the double alpha_0 argument to the algorithm
* @return an initialized DirichletState
*/
- public static DirichletState<VectorWritable> createState(String modelFactory, String modelPrototype, int prototypeSize,
- int numModels, double alpha_0) throws ClassNotFoundException, InstantiationException, IllegalAccessException,
- SecurityException, NoSuchMethodException, IllegalArgumentException, InvocationTargetException {
+ public static DirichletState<VectorWritable> createState(String modelFactory,
+ String modelPrototype,
+ int prototypeSize,
+ int numModels,
+ double alpha0)
+ throws ClassNotFoundException, InstantiationException, IllegalAccessException,
+ SecurityException, NoSuchMethodException, IllegalArgumentException, InvocationTargetException {
ClassLoader ccl = Thread.currentThread().getContextClassLoader();
Class<? extends VectorModelDistribution> cl = ccl.loadClass(modelFactory).asSubclass(VectorModelDistribution.class);
@@ -238,7 +267,7 @@ public final class DirichletDriver {
Class<? extends Vector> vcl = ccl.loadClass(modelPrototype).asSubclass(Vector.class);
Constructor<? extends Vector> v = vcl.getConstructor(int.class);
factory.setModelPrototype(new VectorWritable(v.newInstance(prototypeSize)));
- return new DirichletState<VectorWritable>(factory, numModels, alpha_0);
+ return new DirichletState<VectorWritable>(factory, numModels, alpha0);
}
/**
@@ -258,13 +287,20 @@ public final class DirichletDriver {
* the size of the model prototype vector
* @param numClusters
* the number of clusters
- * @param alpha_0
+ * @param alpha0
* alpha_0
* @param numReducers
* the number of Reducers desired
*/
- public static void runIteration(Path input, Path stateIn, Path stateOut, String modelFactory, String modelPrototype,
- int prototypeSize, int numClusters, double alpha_0, int numReducers) {
+ public static void runIteration(Path input,
+ Path stateIn,
+ Path stateOut,
+ String modelFactory,
+ String modelPrototype,
+ int prototypeSize,
+ int numClusters,
+ double alpha0,
+ int numReducers) {
Configurable client = new JobClient();
JobConf conf = new JobConf(DirichletDriver.class);
@@ -286,7 +322,7 @@ public final class DirichletDriver {
conf.set(MODEL_PROTOTYPE_KEY, modelPrototype);
conf.set(PROTOTYPE_SIZE_KEY, Integer.toString(prototypeSize));
conf.set(NUM_CLUSTERS_KEY, Integer.toString(numClusters));
- conf.set(ALPHA_0_KEY, Double.toString(alpha_0));
+ conf.set(ALPHA_0_KEY, Double.toString(alpha0));
client.setConf(conf);
try {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java Fri May 28 20:03:36 2010
@@ -77,19 +77,17 @@ public class DirichletMapper extends Map
}
}
- public static DirichletState<VectorWritable> getDirichletState(JobConf job) throws SecurityException,
- IllegalArgumentException,
- NoSuchMethodException,
- InvocationTargetException {
+ public static DirichletState<VectorWritable> getDirichletState(JobConf job)
+ throws SecurityException, IllegalArgumentException, NoSuchMethodException, InvocationTargetException {
String statePath = job.get(DirichletDriver.STATE_IN_KEY);
String modelFactory = job.get(DirichletDriver.MODEL_FACTORY_KEY);
String modelPrototype = job.get(DirichletDriver.MODEL_PROTOTYPE_KEY);
String prototypeSize = job.get(DirichletDriver.PROTOTYPE_SIZE_KEY);
String numClusters = job.get(DirichletDriver.NUM_CLUSTERS_KEY);
- String alpha_0 = job.get(DirichletDriver.ALPHA_0_KEY);
+ String alpha0 = job.get(DirichletDriver.ALPHA_0_KEY);
try {
- double alpha = Double.parseDouble(alpha_0);
+ double alpha = Double.parseDouble(alpha0);
DirichletState<VectorWritable> state = DirichletDriver.createState(modelFactory, modelPrototype,
Integer.parseInt(prototypeSize), Integer.parseInt(numClusters), alpha);
Path path = new Path(statePath);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletReducer.java Fri May 28 20:03:36 2010
@@ -44,8 +44,10 @@ public class DirichletReducer extends Ma
}
@Override
- public void reduce(Text key, Iterator<VectorWritable> values, OutputCollector<Text, DirichletCluster<VectorWritable>> output,
- Reporter reporter) throws IOException {
+ public void reduce(Text key,
+ Iterator<VectorWritable> values,
+ OutputCollector<Text, DirichletCluster<VectorWritable>> output,
+ Reporter reporter) throws IOException {
this.output = output;
int k = Integer.parseInt(key.toString());
Model<VectorWritable> model = newModels[k];
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletState.java Fri May 28 20:03:36 2010
@@ -35,21 +35,21 @@ public class DirichletState<O> {
private Vector mixture; // the mixture vector
- private double alpha_0; // alpha_0
+ private double alpha0; // alpha0
public DirichletState(ModelDistribution<O> modelFactory,
int numClusters,
- double alpha_0) {
+ double alpha0) {
this.numClusters = numClusters;
this.modelFactory = modelFactory;
- this.alpha_0 = alpha_0;
+ this.alpha0 = alpha0;
// sample initial prior models
clusters = new ArrayList<DirichletCluster<O>>();
for (Model<O> m : modelFactory.sampleFromPrior(numClusters)) {
clusters.add(new DirichletCluster<O>(m));
}
// sample the mixture parameters from a Dirichlet distribution on the totalCounts
- mixture = UncommonDistributions.rDirichlet(totalCounts(), alpha_0);
+ mixture = UncommonDistributions.rDirichlet(totalCounts(), alpha0);
}
public DirichletState() { }
@@ -107,7 +107,7 @@ public class DirichletState<O> {
clusters.get(i).setModel(newModels[i]);
}
// update the mixture
- mixture = UncommonDistributions.rDirichlet(totalCounts(), alpha_0);
+ mixture = UncommonDistributions.rDirichlet(totalCounts(), alpha0);
}
/**
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/UncommonDistributions.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/UncommonDistributions.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/UncommonDistributions.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/UncommonDistributions.java Fri May 28 20:03:36 2010
@@ -244,18 +244,18 @@ public final class UncommonDistributions
*
* @param totalCounts
* an unnormalized count Vector
- * @param alpha_0
+ * @param alpha0
* a double
* @return a Vector of probabilities
*/
- public static Vector rDirichlet(Vector totalCounts, double alpha_0) {
+ public static Vector rDirichlet(Vector totalCounts, double alpha0) {
Vector pi = totalCounts.like();
double total = totalCounts.zSum();
double remainder = 1.0;
for (int k = 0; k < pi.size(); k++) {
double countK = totalCounts.get(k);
total -= countK;
- double betaK = rBeta(1 + countK, Math.max(0, alpha_0 + total));
+ double betaK = rBeta(1.0 + countK, Math.max(0.0, alpha0 + total));
double piK = betaK * remainder;
pi.set(k, piK);
remainder -= piK;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java Fri May 28 20:03:36 2010
@@ -34,8 +34,9 @@ import com.google.gson.reflect.TypeToken
public class AsymmetricSampledNormalModel implements Model<VectorWritable> {
- private static final double sqrt2pi = Math.sqrt(2.0 * Math.PI);
-
+ private static final double SQRT2PI = Math.sqrt(2.0 * Math.PI);
+ private static final Type MODEL_TYPE = new TypeToken<Model<Vector>>() { }.getType();
+
private int id;
// the parameters
@@ -44,20 +45,16 @@ public class AsymmetricSampledNormalMode
private Vector stdDev;
// the observation statistics, initialized by the first observation
- private int s0 = 0;
+ private int s0;
private Vector s1;
private Vector s2;
-
- private static final Type modelType = new TypeToken<Model<Vector>>() { }.getType();
-
+
public AsymmetricSampledNormalModel() {
- super();
}
public AsymmetricSampledNormalModel(int id, Vector mean, Vector stdDev) {
- super();
this.id = id;
this.mean = mean;
this.stdDev = stdDev;
@@ -125,7 +122,7 @@ public class AsymmetricSampledNormalMode
double sd2 = sd * sd;
double exp = -(x.dot(x) - 2 * x.dot(mean) + mean.dot(mean)) / (2 * sd2);
double ex = Math.exp(exp);
- return ex / (sd * sqrt2pi);
+ return ex / (sd * SQRT2PI);
}
@Override
@@ -195,7 +192,7 @@ public class AsymmetricSampledNormalMode
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Model.class, new JsonModelAdapter());
Gson gson = builder.create();
- return gson.toJson(this, modelType);
+ return gson.toJson(this, MODEL_TYPE);
}
@Override
@@ -205,7 +202,7 @@ public class AsymmetricSampledNormalMode
@Override
public int getId() {
- return id;
+ return id;
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java Fri May 28 20:03:36 2010
@@ -34,12 +34,18 @@ import com.google.gson.reflect.TypeToken
public class L1Model implements Model<VectorWritable> {
- private static final DistanceMeasure measure = new ManhattanDistanceMeasure();
+ private static final DistanceMeasure MEASURE = new ManhattanDistanceMeasure();
+ private static final Type MODEL_TYPE = new TypeToken<Model<Vector>>() {}.getType();
private int id;
+ private Vector coefficients;
+
+ private int count;
+
+ private Vector observed;
+
public L1Model() {
- super();
}
public L1Model(int id, Vector v) {
@@ -48,15 +54,6 @@ public class L1Model implements Model<Ve
coefficients = v;
}
- private Vector coefficients;
-
- private int count = 0;
-
- private Vector observed;
-
- private static final Type modelType = new TypeToken<Model<Vector>>() {
- }.getType();
-
@Override
public void computeParameters() {
coefficients = observed.divide(count);
@@ -75,7 +72,7 @@ public class L1Model implements Model<Ve
@Override
public double pdf(VectorWritable x) {
- return Math.exp(-L1Model.measure.distance(x.get(), coefficients));
+ return Math.exp(-L1Model.MEASURE.distance(x.get(), coefficients));
}
@Override
@@ -125,7 +122,7 @@ public class L1Model implements Model<Ve
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Model.class, new JsonModelAdapter());
Gson gson = builder.create();
- return gson.toJson(this, modelType);
+ return gson.toJson(this, MODEL_TYPE);
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java Fri May 28 20:03:36 2010
@@ -35,8 +35,10 @@ import com.google.gson.reflect.TypeToken
public class NormalModel implements Model<VectorWritable> {
- private static final double sqrt2pi = Math.sqrt(2.0 * Math.PI);
-
+ private static final double SQRT2PI = Math.sqrt(2.0 * Math.PI);
+
+ private static final Type MODEL_TYPE = new TypeToken<Model<Vector>>() {}.getType();
+
private int id;
// the parameters
@@ -45,14 +47,12 @@ public class NormalModel implements Mode
private double stdDev;
// the observation statistics, initialized by the first observation
- private int s0 = 0;
+ private int s0;
private Vector s1;
private Vector s2;
-
- private static final Type modelType = new TypeToken<Model<Vector>>() { }.getType();
-
+
public NormalModel() { }
public NormalModel(int id, Vector mean, double stdDev) {
@@ -122,7 +122,7 @@ public class NormalModel implements Mode
double sd2 = stdDev * stdDev;
double exp = -(x.dot(x) - 2 * x.dot(mean) + mean.dot(mean)) / (2 * sd2);
double ex = Math.exp(exp);
- return ex / (stdDev * sqrt2pi);
+ return ex / (stdDev * SQRT2PI);
}
@Override
@@ -175,7 +175,7 @@ public class NormalModel implements Mode
GsonBuilder builder = new GsonBuilder();
builder.registerTypeAdapter(Model.class, new JsonModelAdapter());
Gson gson = builder.create();
- return gson.toJson(this, modelType);
+ return gson.toJson(this, MODEL_TYPE);
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/VectorModelDistribution.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/VectorModelDistribution.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/VectorModelDistribution.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/VectorModelDistribution.java Fri May 28 20:03:36 2010
@@ -19,20 +19,18 @@ package org.apache.mahout.clustering.dir
import org.apache.mahout.math.VectorWritable;
public abstract class VectorModelDistribution implements ModelDistribution<VectorWritable> {
-
+
+ // a prototype instance used for creating prior model distributions using like(). It
+ // should be of the class and cardinality desired for the particular application.
+ private VectorWritable modelPrototype;
+
protected VectorModelDistribution() {
- super();
}
protected VectorModelDistribution(VectorWritable modelPrototype) {
- super();
this.modelPrototype = modelPrototype;
}
-
- // a prototype instance used for creating prior model distributions using like(). It
- // should be of the class and cardinality desired for the particular application.
- private VectorWritable modelPrototype;
-
+
/**
* @return the modelPrototype
*/
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterMapper.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterMapper.java Fri May 28 20:03:36 2010
@@ -44,8 +44,10 @@ public class FuzzyKMeansClusterMapper ex
//private double threshold = 0;
@Override
- public void map(WritableComparable<?> key, VectorWritable point, OutputCollector<IntWritable, WeightedVectorWritable> output,
- Reporter reporter) throws IOException {
+ public void map(WritableComparable<?> key,
+ VectorWritable point,
+ OutputCollector<IntWritable, WeightedVectorWritable> output,
+ Reporter reporter) throws IOException {
clusterer.emitPointToClusters(point, clusters, output);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java Fri May 28 20:03:36 2010
@@ -43,7 +43,7 @@ public class FuzzyKMeansClusterer {
private boolean emitMostLikely = true;
- private double threshold = 0;
+ private double threshold;
/**
* Init the fuzzy k-means clusterer with the distance measure to use for comparison.
@@ -101,8 +101,9 @@ public class FuzzyKMeansClusterer {
* @param output
* the OutputCollector to emit into
*/
- public void emitPointProbToCluster(Vector point, List<SoftCluster> clusters, OutputCollector<Text, FuzzyKMeansInfo> output)
- throws IOException {
+ public void emitPointProbToCluster(Vector point,
+ List<SoftCluster> clusters,
+ OutputCollector<Text, FuzzyKMeansInfo> output) throws IOException {
List<Double> clusterDistanceList = new ArrayList<Double>();
for (SoftCluster cluster : clusters) {
@@ -240,8 +241,12 @@ public class FuzzyKMeansClusterer {
* @return
* a List<List<SoftCluster>> of clusters produced per iteration
*/
- public static List<List<SoftCluster>> clusterPoints(List<Vector> points, List<SoftCluster> clusters, DistanceMeasure measure,
- double threshold, double m, int numIter) {
+ public static List<List<SoftCluster>> clusterPoints(List<Vector> points,
+ List<SoftCluster> clusters,
+ DistanceMeasure measure,
+ double threshold,
+ double m,
+ int numIter) {
List<List<SoftCluster>> clustersList = new ArrayList<List<SoftCluster>>();
clustersList.add(clusters);
FuzzyKMeansClusterer clusterer = new FuzzyKMeansClusterer(measure, threshold, m);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java Fri May 28 20:03:36 2010
@@ -42,8 +42,7 @@ public class FuzzyKMeansCombiner extends
// String pointInfo = values.next().toString();
FuzzyKMeansInfo info = values.next();
- if (info.getCombinerPass() == 0) // first time thru combiner
- {
+ if (info.getCombinerPass() == 0) { // first time thru combiner
cluster.addPoint(info.getVector(), Math.pow(info.getProbability(), clusterer.getM()));
} else {
cluster.addPoints(info.getVector(), info.getProbability());
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java Fri May 28 20:03:36 2010
@@ -65,7 +65,8 @@ public final class FuzzyKMeansDriver {
Option measureClassOpt = DefaultOptionCreator.distanceMeasureOption().create();
Option clustersOpt = DefaultOptionCreator.clustersInOption().withDescription(
"The input centroids, as Vectors. Must be a SequenceFile of Writable, Cluster/Canopy. "
- + "If k is also specified, then a random set of vectors will be selected" + " and written out to this path first")
+ + "If k is also specified, then a random set of vectors will be selected"
+ + " and written out to this path first")
.create();
Option kOpt = DefaultOptionCreator.kOption().withDescription(
"The k in k-Means. If specified, then a random selection of k Vectors will be chosen"
@@ -81,10 +82,11 @@ public final class FuzzyKMeansDriver {
Option emitMostLikelyOpt = DefaultOptionCreator.emitMostLikelyOption().create();
Option thresholdOpt = DefaultOptionCreator.thresholdOption().create();
- Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(clustersOpt).withOption(outputOpt)
- .withOption(measureClassOpt).withOption(convergenceDeltaOpt).withOption(maxIterationsOpt).withOption(kOpt).withOption(mOpt)
- .withOption(overwriteOutput).withOption(helpOpt).withOption(numMapTasksOpt).withOption(numReduceTasksOpt).withOption(
- emitMostLikelyOpt).withOption(thresholdOpt).create();
+ Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(clustersOpt)
+ .withOption(outputOpt).withOption(measureClassOpt).withOption(convergenceDeltaOpt)
+ .withOption(maxIterationsOpt).withOption(kOpt).withOption(mOpt)
+ .withOption(overwriteOutput).withOption(helpOpt).withOption(numMapTasksOpt)
+ .withOption(numReduceTasksOpt).withOption(emitMostLikelyOpt).withOption(thresholdOpt).create();
try {
Parser parser = new Parser();
@@ -114,10 +116,21 @@ public final class FuzzyKMeansDriver {
boolean emitMostLikely = Boolean.parseBoolean(cmdLine.getValue(emitMostLikelyOpt).toString());
double threshold = Double.parseDouble(cmdLine.getValue(thresholdOpt).toString());
if (cmdLine.hasOption(kOpt)) {
- clusters = RandomSeedGenerator.buildRandom(input, clusters, Integer.parseInt(cmdLine.getValue(kOpt).toString()));
+ clusters = RandomSeedGenerator.buildRandom(input, clusters,
+ Integer.parseInt(cmdLine.getValue(kOpt).toString()));
}
- runJob(input, clusters, output, measureClass, convergenceDelta, maxIterations, numMapTasks, numReduceTasks, m, cmdLine
- .hasOption(clusteringOpt), emitMostLikely, threshold);
+ runJob(input,
+ clusters,
+ output,
+ measureClass,
+ convergenceDelta,
+ maxIterations,
+ numMapTasks,
+ numReduceTasks,
+ m,
+ cmdLine.hasOption(clusteringOpt),
+ emitMostLikely,
+ threshold);
} catch (OptionException e) {
log.error("Exception", e);
@@ -155,9 +168,18 @@ public final class FuzzyKMeansDriver {
* @param threshold
* a double threshold value emits all clusters having greater pdf (emitMostLikely = false)
*/
- public static void runJob(Path input, Path clustersIn, Path output, String measureClass, double convergenceDelta,
- int maxIterations, int numMapTasks, int numReduceTasks, float m, boolean runClustering, boolean emitMostLikely,
- double threshold) {
+ public static void runJob(Path input,
+ Path clustersIn,
+ Path output,
+ String measureClass,
+ double convergenceDelta,
+ int maxIterations,
+ int numMapTasks,
+ int numReduceTasks,
+ float m,
+ boolean runClustering,
+ boolean emitMostLikely,
+ double threshold) {
boolean converged = false;
int iteration = 1;
@@ -168,8 +190,15 @@ public final class FuzzyKMeansDriver {
// point the output to a new directory per iteration
Path clustersOut = new Path(output, Cluster.CLUSTERS_DIR + iteration);
- converged = runIteration(input, clustersIn, clustersOut, measureClass, convergenceDelta, numMapTasks, numReduceTasks,
- iteration, m);
+ converged = runIteration(input,
+ clustersIn,
+ clustersOut,
+ measureClass,
+ convergenceDelta,
+ numMapTasks,
+ numReduceTasks,
+ iteration,
+ m);
// now point the input to the old output directory
clustersIn = clustersOut;
@@ -178,8 +207,15 @@ public final class FuzzyKMeansDriver {
// now actually cluster the points
log.info("Clustering ");
- runClustering(input, clustersIn, new Path(output, Cluster.CLUSTERED_POINTS_DIR), measureClass, convergenceDelta, numMapTasks,
- m, emitMostLikely, threshold);
+ runClustering(input,
+ clustersIn,
+ new Path(output, Cluster.CLUSTERED_POINTS_DIR),
+ measureClass,
+ convergenceDelta,
+ numMapTasks,
+ m,
+ emitMostLikely,
+ threshold);
}
/**
@@ -204,8 +240,15 @@ public final class FuzzyKMeansDriver {
* http://en.wikipedia.org/wiki/Data_clustering#Fuzzy_c-means_clustering
* @return true if the iteration successfully runs
*/
- private static boolean runIteration(Path input, Path clustersIn, Path clustersOut, String measureClass, double convergenceDelta,
- int numMapTasks, int numReduceTasks, int iterationNumber, float m) {
+ private static boolean runIteration(Path input,
+ Path clustersIn,
+ Path clustersOut,
+ String measureClass,
+ double convergenceDelta,
+ int numMapTasks,
+ int numReduceTasks,
+ int iterationNumber,
+ float m) {
JobConf conf = new JobConf(FuzzyKMeansDriver.class);
conf.setJobName("Fuzzy K Means{" + iterationNumber + '}');
@@ -268,8 +311,15 @@ public final class FuzzyKMeansDriver {
* @param threshold
* a double threshold value emits all clusters having greater pdf (emitMostLikely = false)
*/
- private static void runClustering(Path input, Path clustersIn, Path output, String measureClass, double convergenceDelta,
- int numMapTasks, float m, boolean emitMostLikely, double threshold) {
+ private static void runClustering(Path input,
+ Path clustersIn,
+ Path output,
+ String measureClass,
+ double convergenceDelta,
+ int numMapTasks,
+ float m,
+ boolean emitMostLikely,
+ double threshold) {
JobConf conf = new JobConf(FuzzyKMeansDriver.class);
conf.setJobName("Fuzzy K Means Clustering");
@@ -327,7 +377,8 @@ public final class FuzzyKMeansDriver {
}
};
- FileStatus[] matches = fs.listStatus(FileUtil.stat2Paths(fs.globStatus(clusterPath, clusterFileFilter)), clusterFileFilter);
+ FileStatus[] matches = fs.listStatus(FileUtil.stat2Paths(
+ fs.globStatus(clusterPath, clusterFileFilter)), clusterFileFilter);
for (FileStatus match : matches) {
result.add(fs.makeQualified(match.getPath()));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java Fri May 28 20:03:36 2010
@@ -49,8 +49,7 @@ public class FuzzyKMeansReducer extends
while (values.hasNext()) {
FuzzyKMeansInfo value = values.next();
- if (value.getCombinerPass() == 0) // escaped from combiner
- {
+ if (value.getCombinerPass() == 0) { // escaped from combiner
cluster.addPoint(value.getVector(), Math.pow(value.getProbability(), clusterer.getM()));
} else {
cluster.addPoints(value.getVector(), value.getProbability());
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java Fri May 28 20:03:36 2010
@@ -58,7 +58,8 @@ final class FuzzyKMeansUtil {
try {
// get all filtered file names in result list
FileSystem fs = clusterPath.getFileSystem(job);
- FileStatus[] matches = fs.listStatus(FileUtil.stat2Paths(fs.globStatus(clusterPath, clusterFileFilter)), clusterFileFilter);
+ FileStatus[] matches = fs.listStatus(FileUtil.stat2Paths(
+ fs.globStatus(clusterPath, clusterFileFilter)), clusterFileFilter);
for (FileStatus match : matches) {
result.add(fs.makeQualified(match.getPath()));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java Fri May 28 20:03:36 2010
@@ -50,9 +50,47 @@ public class SoftCluster extends Cluster
private Vector s2;
+ // For Writable
+ public SoftCluster() {
+ }
+
/**
- * Format the SoftCluster for output
+ * Construct a new SoftCluster with the given point as its center
*
+ * @param center
+ * the center point
+ */
+ public SoftCluster(Vector center) {
+ setCenter(center.clone());
+ this.pointProbSum = 0;
+ this.weightedPointTotal = getCenter().like();
+ }
+
+ /**
+ * Construct a new SoftCluster with the given point as its center
+ *
+ * @param center
+ * the center point
+ */
+ public SoftCluster(Vector center, int clusterId) {
+ this.setId(clusterId);
+ this.setCenter(new RandomAccessSparseVector(center));
+ this.pointProbSum = 0;
+ this.weightedPointTotal = center.like();
+ }
+
+ /** Construct a new softcluster with the given clusterID */
+ public SoftCluster(String clusterId) {
+
+ this.setId(Integer.parseInt(clusterId.substring(1)));
+ this.pointProbSum = 0;
+ // this.weightedPointTotal = center.like();
+ this.converged = clusterId.charAt(0) == 'V';
+ }
+
+ /**
+ * Format the SoftCluster for output
+ *
* @param cluster
* the Cluster
*/
@@ -62,7 +100,7 @@ public class SoftCluster extends Cluster
/**
* Decodes and returns a SoftCluster from the formattedString
- *
+ *
* @param formattedString
* a String produced by formatCluster
*/
@@ -83,21 +121,6 @@ public class SoftCluster extends Cluster
return null;
}
- // For Writable
- public SoftCluster() {
- }
-
- /**
- * Construct a new SoftCluster with the given point as its center
- *
- * @param center
- * the center point
- */
- public SoftCluster(Vector center) {
- setCenter(center.clone());
- this.pointProbSum = 0;
- this.weightedPointTotal = getCenter().like();
- }
@Override
public void write(DataOutput out) throws IOException {
@@ -135,28 +158,6 @@ public class SoftCluster extends Cluster
return centroid;
}
- /**
- * Construct a new SoftCluster with the given point as its center
- *
- * @param center
- * the center point
- */
- public SoftCluster(Vector center, int clusterId) {
- this.setId(clusterId);
- this.setCenter(new RandomAccessSparseVector(center));
- this.pointProbSum = 0;
- this.weightedPointTotal = center.like();
- }
-
- /** Construct a new softcluster with the given clusterID */
- public SoftCluster(String clusterId) {
-
- this.setId(Integer.parseInt(clusterId.substring(1)));
- this.pointProbSum = 0;
- // this.weightedPointTotal = center.like();
- this.converged = clusterId.charAt(0) == 'V';
- }
-
@Override
public String toString() {
return asFormatString(null);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java Fri May 28 20:03:36 2010
@@ -44,6 +44,46 @@ public class Cluster extends ClusterBase
private boolean converged;
private double std = 0.00000001;
+
+ /**
+ * Construct a new cluster with the given point as its center
+ *
+ * @param center
+ * the center point
+ */
+ public Cluster(Vector center) {
+ super();
+ this.setCenter(new RandomAccessSparseVector(center));
+ this.setNumPoints(0);
+ this.setPointTotal(getCenter().like());
+ this.pointSquaredTotal = getCenter().like();
+ }
+
+ /** For (de)serialization as a Writable */
+ public Cluster() { }
+
+ /**
+ * Construct a new cluster with the given point as its center
+ *
+ * @param center
+ * the center point
+ */
+ public Cluster(Vector center, int clusterId) {
+ super();
+ this.setId(clusterId);
+ this.setCenter(new RandomAccessSparseVector(center));
+ this.setNumPoints(0);
+ this.setPointTotal(getCenter().like());
+ this.pointSquaredTotal = getCenter().like();
+ }
+
+ /** Construct a new clsuter with the given id as identifier */
+ public Cluster(String clusterId) {
+
+ this.setId(Integer.parseInt(clusterId.substring(1)));
+ this.setNumPoints(0);
+ this.converged = clusterId.startsWith("V");
+ }
/**
* Format the cluster for output
@@ -126,46 +166,6 @@ public class Cluster extends ClusterBase
return centroid;
}
- /**
- * Construct a new cluster with the given point as its center
- *
- * @param center
- * the center point
- */
- public Cluster(Vector center) {
- super();
- this.setCenter(new RandomAccessSparseVector(center));
- this.setNumPoints(0);
- this.setPointTotal(getCenter().like());
- this.pointSquaredTotal = getCenter().like();
- }
-
- /** For (de)serialization as a Writable */
- public Cluster() { }
-
- /**
- * Construct a new cluster with the given point as its center
- *
- * @param center
- * the center point
- */
- public Cluster(Vector center, int clusterId) {
- super();
- this.setId(clusterId);
- this.setCenter(new RandomAccessSparseVector(center));
- this.setNumPoints(0);
- this.setPointTotal(getCenter().like());
- this.pointSquaredTotal = getCenter().like();
- }
-
- /** Construct a new clsuter with the given id as identifier */
- public Cluster(String clusterId) {
-
- this.setId(Integer.parseInt(clusterId.substring(1)));
- this.setNumPoints(0);
- this.converged = clusterId.startsWith("V");
- }
-
@Override
public String toString() {
return getIdentifier() + ": " + getCenter().asFormatString();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java Fri May 28 20:03:36 2010
@@ -61,8 +61,9 @@ public class KMeansClusterer {
* @param clusters
* a List<Cluster> to test.
*/
- public void emitPointToNearestCluster(Vector point, List<Cluster> clusters, OutputCollector<Text, KMeansInfo> output)
- throws IOException {
+ public void emitPointToNearestCluster(Vector point,
+ Iterable<Cluster> clusters,
+ OutputCollector<Text, KMeansInfo> output) throws IOException {
Cluster nearestCluster = null;
double nearestDistance = Double.MAX_VALUE;
for (Cluster cluster : clusters) {
@@ -80,7 +81,7 @@ public class KMeansClusterer {
output.collect(new Text(nearestCluster.getIdentifier()), new KMeansInfo(1, point));
}
- public void outputPointWithClusterInfo(Vector vector, List<Cluster> clusters,
+ public void outputPointWithClusterInfo(Vector vector, Iterable<Cluster> clusters,
OutputCollector<IntWritable, WeightedVectorWritable> output) throws IOException {
Cluster nearestCluster = null;
double nearestDistance = Double.MAX_VALUE;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java Fri May 28 20:03:36 2010
@@ -56,7 +56,8 @@ public final class KMeansDriver {
Option inputOpt = DefaultOptionCreator.inputOption().create();
Option clustersOpt = DefaultOptionCreator.clustersInOption().withDescription(
"The input centroids, as Vectors. Must be a SequenceFile of Writable, Cluster/Canopy. "
- + "If k is also specified, then a random set of vectors will be selected" + " and written out to this path first")
+ + "If k is also specified, then a random set of vectors will be selected"
+ + " and written out to this path first")
.create();
Option kOpt = DefaultOptionCreator.kOption().withDescription(
"The k in k-Means. If specified, then a random selection of k Vectors will be chosen"
@@ -70,8 +71,9 @@ public final class KMeansDriver {
Option clusteringOpt = DefaultOptionCreator.clusteringOption().create();
Option helpOpt = DefaultOptionCreator.helpOption();
- Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(clustersOpt).withOption(outputOpt)
- .withOption(measureClassOpt).withOption(convergenceDeltaOpt).withOption(maxIterationsOpt).withOption(numReduceTasksOpt)
+ Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(clustersOpt)
+ .withOption(outputOpt).withOption(measureClassOpt).withOption(convergenceDeltaOpt)
+ .withOption(maxIterationsOpt).withOption(numReduceTasksOpt)
.withOption(kOpt).withOption(overwriteOutput).withOption(helpOpt).withOption(clusteringOpt).create();
try {
Parser parser = new Parser();
@@ -94,7 +96,8 @@ public final class KMeansDriver {
HadoopUtil.overwriteOutput(output);
}
if (cmdLine.hasOption(kOpt)) {
- clusters = RandomSeedGenerator.buildRandom(input, clusters, Integer.parseInt(cmdLine.getValue(kOpt).toString()));
+ clusters = RandomSeedGenerator.buildRandom(input, clusters,
+ Integer.parseInt(cmdLine.getValue(kOpt).toString()));
}
runJob(input, clusters, output, measureClass, convergenceDelta, maxIterations, numReduceTasks, cmdLine
.hasOption(clusteringOpt));
@@ -129,9 +132,10 @@ public final class KMeansDriver {
// iterate until the clusters converge
String delta = Double.toString(convergenceDelta);
if (log.isInfoEnabled()) {
- log.info("Input: {} Clusters In: {} Out: {} Distance: {}", new Object[] { input, clustersIn, output, measureClass });
- log.info("convergence: {} max Iterations: {} num Reduce Tasks: {} Input Vectors: {}", new Object[] { convergenceDelta,
- maxIterations, numReduceTasks, VectorWritable.class.getName() });
+ log.info("Input: {} Clusters In: {} Out: {} Distance: {}",
+ new Object[] {input, clustersIn, output, measureClass});
+ log.info("convergence: {} max Iterations: {} num Reduce Tasks: {} Input Vectors: {}",
+ new Object[] {convergenceDelta, maxIterations, numReduceTasks, VectorWritable.class.getName()});
}
boolean converged = false;
int iteration = 1;
@@ -168,8 +172,12 @@ public final class KMeansDriver {
* the number of reducer tasks
* @return true if the iteration successfully runs
*/
- private static boolean runIteration(Path input, Path clustersIn, Path clustersOut, String measureClass, String convergenceDelta,
- int numReduceTasks) throws IOException {
+ private static boolean runIteration(Path input,
+ Path clustersIn,
+ Path clustersOut,
+ String measureClass,
+ String convergenceDelta,
+ int numReduceTasks) throws IOException {
JobConf conf = new JobConf(KMeansDriver.class);
conf.setMapOutputKeyClass(Text.class);
conf.setMapOutputValueClass(KMeansInfo.class);
@@ -213,11 +221,15 @@ public final class KMeansDriver {
* @param convergenceDelta
* the convergence delta value
*/
- private static void runClustering(Path input, Path clustersIn, Path output, String measureClass, String convergenceDelta)
- throws IOException {
+ private static void runClustering(Path input,
+ Path clustersIn,
+ Path output,
+ String measureClass,
+ String convergenceDelta) throws IOException {
if (log.isInfoEnabled()) {
log.info("Running Clustering");
- log.info("Input: {} Clusters In: {} Out: {} Distance: {}", new Object[] { input, clustersIn, output, measureClass });
+ log.info("Input: {} Clusters In: {} Out: {} Distance: {}",
+ new Object[] {input, clustersIn, output, measureClass});
log.info("convergence: {} Input Vectors: {}", convergenceDelta, VectorWritable.class.getName());
}
JobConf conf = new JobConf(KMeansDriver.class);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java Fri May 28 20:03:36 2010
@@ -48,7 +48,8 @@ public final class RandomSeedGenerator {
public static final String K = "k";
- private RandomSeedGenerator() {}
+ private RandomSeedGenerator() {
+ }
public static Path buildRandom(Path input, Path output, int k) throws IOException,
IllegalAccessException,
@@ -111,10 +112,10 @@ public final class RandomSeedGenerator {
return outFile;
}
- public static List<Vector> chooseRandomPoints(List<Vector> vectors, int k) {
+ public static List<Vector> chooseRandomPoints(Iterable<Vector> vectors, int k) {
List<Vector> chosenPoints = new ArrayList<Vector>(k);
Random random = RandomUtils.getRandom();
- for(Vector value : vectors){
+ for (Vector value : vectors) {
int currentSize = chosenPoints.size();
if (currentSize < k) {
chosenPoints.add(value);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java Fri May 28 20:03:36 2010
@@ -82,9 +82,9 @@ public final class LDADriver {
Option numReducOpt = DefaultOptionCreator.numReducersOption().create();
Option helpOpt = DefaultOptionCreator.helpOption();
- Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(outputOpt).withOption(topicsOpt)
- .withOption(wordsOpt).withOption(topicSmOpt).withOption(maxIterOpt).withOption(numReducOpt).withOption(overwriteOutput)
- .withOption(helpOpt).create();
+ Group group = new GroupBuilder().withName("Options").withOption(inputOpt).withOption(outputOpt)
+ .withOption(topicsOpt).withOption(wordsOpt).withOption(topicSmOpt).withOption(maxIterOpt)
+ .withOption(numReducOpt).withOption(overwriteOutput).withOption(helpOpt).create();
try {
Parser parser = new Parser();
parser.setGroup(group);
@@ -136,8 +136,13 @@ public final class LDADriver {
* the number of Reducers desired
* @throws IOException
*/
- public static void runJob(Path input, Path output, int numTopics, int numWords, double topicSmoothing, int maxIterations,
- int numReducers) throws IOException, InterruptedException, ClassNotFoundException {
+ public static void runJob(Path input,
+ Path output,
+ int numTopics,
+ int numWords,
+ double topicSmoothing,
+ int maxIterations,
+ int numReducers) throws IOException, InterruptedException, ClassNotFoundException {
Path stateIn = new Path(output, "state-0");
writeInitialState(stateIn, numTopics, numWords);
@@ -227,8 +232,13 @@ public final class LDADriver {
* @param numReducers
* the number of Reducers desired
*/
- public static double runIteration(Path input, Path stateIn, Path stateOut, int numTopics, int numWords, double topicSmoothing,
- int numReducers) throws IOException, InterruptedException, ClassNotFoundException {
+ public static double runIteration(Path input,
+ Path stateIn,
+ Path stateOut,
+ int numTopics,
+ int numWords,
+ double topicSmoothing,
+ int numReducers) throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set(STATE_IN_KEY, stateIn.toString());
conf.set(NUM_TOPICS_KEY, Integer.toString(numTopics));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAInference.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAInference.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAInference.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAInference.java Fri May 28 20:03:36 2010
@@ -34,7 +34,10 @@ public class LDAInference {
private static final double E_STEP_CONVERGENCE = 1.0E-6;
private static final int MAX_ITER = 20;
-
+
+ private DenseMatrix phi;
+ private final LDAState state;
+
public LDAInference(LDAState state) {
this.state = state;
}
@@ -49,12 +52,8 @@ public class LDAInference {
private final Vector gamma; // p(topic)
private final Matrix mphi; // log p(columnMap(w)|t)
private final int[] columnMap; // maps words into the matrix's column map
- public final double logLikelihood;
-
- public double phi(int k, int w) {
- return mphi.getQuick(k, columnMap[w]);
- }
-
+ private final double logLikelihood;
+
InferredDocument(Vector wordCounts, Vector gamma, int[] columnMap, Matrix phi, double ll) {
this.wordCounts = wordCounts;
this.gamma = gamma;
@@ -62,6 +61,10 @@ public class LDAInference {
this.columnMap = columnMap;
this.logLikelihood = ll;
}
+
+ public double phi(int k, int w) {
+ return mphi.getQuick(k, columnMap[w]);
+ }
public Vector getWordCounts() {
return wordCounts;
@@ -70,6 +73,10 @@ public class LDAInference {
public Vector getGamma() {
return gamma;
}
+
+ public double getLogLikelihood() {
+ return logLikelihood;
+ }
}
/**
@@ -80,9 +87,9 @@ public class LDAInference {
int docLength = wordCounts.size(); // cardinality of document vectors
// initialize variational approximation to p(z|doc)
- Vector gamma = new DenseVector(state.numTopics);
- gamma.assign(state.topicSmoothing + docTotal / state.numTopics);
- Vector nextGamma = new DenseVector(state.numTopics);
+ Vector gamma = new DenseVector(state.getNumTopics());
+ gamma.assign(state.getTopicSmoothing() + docTotal / state.getNumTopics());
+ Vector nextGamma = new DenseVector(state.getNumTopics());
createPhiMatrix(docLength);
Vector digammaGamma = digammaGamma(gamma);
@@ -94,7 +101,7 @@ public class LDAInference {
boolean converged = false;
double oldLL = 1;
while (!converged && (iteration < MAX_ITER)) {
- nextGamma.assign(state.topicSmoothing); // nG := alpha, for all topics
+ nextGamma.assign(state.getTopicSmoothing()); // nG := alpha, for all topics
int mapping = 0;
for (Iterator<Vector.Element> iter = wordCounts.iterateNonZero(); iter.hasNext();) {
@@ -136,7 +143,7 @@ public class LDAInference {
Vector digammaGamma = digamma(gamma);
// and log normalize:
double digammaSumGamma = digamma(gamma.zSum());
- for (int i = 0; i < state.numTopics; i++) {
+ for (int i = 0; i < state.getNumTopics(); i++) {
digammaGamma.setQuick(i, digammaGamma.getQuick(i) - digammaSumGamma);
}
return digammaGamma;
@@ -144,29 +151,26 @@ public class LDAInference {
private void createPhiMatrix(int docLength) {
if (phi == null) {
- phi = new DenseMatrix(state.numTopics, docLength);
+ phi = new DenseMatrix(state.getNumTopics(), docLength);
} else if (phi.getRow(0).size() != docLength) {
- phi = new DenseMatrix(state.numTopics, docLength);
+ phi = new DenseMatrix(state.getNumTopics(), docLength);
} else {
phi.assign(0);
}
}
- private DenseMatrix phi;
- private final LDAState state;
-
private double computeLikelihood(Vector wordCounts, int[] map, Matrix phi, Vector gamma, Vector digammaGamma) {
double ll = 0.0;
// log normalizer for q(gamma);
- ll += Gamma.logGamma(state.topicSmoothing * state.numTopics);
- ll -= state.numTopics * Gamma.logGamma(state.topicSmoothing);
+ ll += Gamma.logGamma(state.getTopicSmoothing() * state.getNumTopics());
+ ll -= state.getNumTopics() * Gamma.logGamma(state.getTopicSmoothing());
// isNotNaNAssertion(ll);
// now for the the rest of q(gamma);
- for (int k = 0; k < state.numTopics; ++k) {
+ for (int k = 0; k < state.getNumTopics(); ++k) {
double gammaK = gamma.get(k);
- ll += (state.topicSmoothing - gammaK) * digammaGamma.getQuick(k);
+ ll += (state.getTopicSmoothing() - gammaK) * digammaGamma.getQuick(k);
ll += Gamma.logGamma(gammaK);
}
@@ -180,7 +184,7 @@ public class LDAInference {
double n = e.get();
int mapping = map[w];
// now for each topic:
- for (int k = 0; k < state.numTopics; k++) {
+ for (int k = 0; k < state.getNumTopics(); k++) {
double llPart = 0.0;
double phiKMapping = phi.getQuick(k, mapping);
llPart += Math.exp(phiKMapping)
@@ -199,16 +203,16 @@ public class LDAInference {
* Compute log q(k|w,doc) for each topic k, for a given word.
*/
private Vector eStepForWord(int word, Vector digammaGamma) {
- Vector phi = new DenseVector(state.numTopics); // log q(k|w), for each w
+ Vector phi = new DenseVector(state.getNumTopics()); // log q(k|w), for each w
double phiTotal = Double.NEGATIVE_INFINITY; // log Normalizer
- for (int k = 0; k < state.numTopics; ++k) { // update q(k|w)'s param phi
+ for (int k = 0; k < state.getNumTopics(); ++k) { // update q(k|w)'s param phi
phi.setQuick(k, state.logProbWordGivenTopic(word, k) + digammaGamma.getQuick(k));
phiTotal = LDAUtil.logSum(phiTotal, phi.getQuick(k));
// assertions(word, digammaGamma, phiTotal, k);
}
- for (int i = 0; i < state.numTopics; i++) {
- phi.setQuick(i, phi.getQuick(i) - phiTotal);// log normalize
+ for (int i = 0; i < state.getNumTopics(); i++) {
+ phi.setQuick(i, phi.getQuick(i) - phiTotal); // log normalize
}
return phi;
}
@@ -250,7 +254,8 @@ public class LDAInference {
}
double f = 1.0 / (x * x);
- double t = f * (-1 / 12.0 + f * (1 / 120.0 + f * (-1 / 252.0 + f * (1 / 240.0 + f * (-1 / 132.0 + f * (691 / 32760.0 + f * (-1 / 12.0 + f * 3617.0 / 8160.0)))))));
+ double t = f * (-1.0 / 12.0 + f * (1.0 / 120.0 + f * (-1.0 / 252.0 + f * (1.0 / 240.0
+ + f * (-1.0 / 132.0 + f * (691.0 / 32760.0 + f * (-1.0 / 12.0 + f * 3617.0 / 8160.0)))))));
return r + Math.log(x) - 0.5 / x + t;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAMapper.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAMapper.java Fri May 28 20:03:36 2010
@@ -47,13 +47,13 @@ public class LDAMapper extends Mapper<Wr
try {
doc = infer.infer(wordCounts);
} catch (ArrayIndexOutOfBoundsException e1) {
- throw new IllegalStateException(
+ throw new IllegalStateException(
"This is probably because the --numWords argument is set too small. \n"
+ "\tIt needs to be >= than the number of words (terms actually) in the corpus and can be \n"
+ "\tlarger if some storage inefficiency can be tolerated.", e1);
}
- double[] logTotals = new double[state.numTopics];
+ double[] logTotals = new double[state.getNumTopics()];
Arrays.fill(logTotals, Double.NEGATIVE_INFINITY);
// Output sufficient statistics for each word. == pseudo-log counts.
@@ -62,7 +62,7 @@ public class LDAMapper extends Mapper<Wr
Vector.Element e = iter.next();
int w = e.index();
- for (int k = 0; k < state.numTopics; ++k) {
+ for (int k = 0; k < state.getNumTopics(); ++k) {
v.set(doc.phi(k, w) + Math.log(e.get()));
IntPairWritable kw = new IntPairWritable(k, w);
@@ -75,7 +75,7 @@ public class LDAMapper extends Mapper<Wr
// Output the totals for the statistics. This is to make
// normalizing a lot easier.
- for (int k = 0; k < state.numTopics; ++k) {
+ for (int k = 0; k < state.getNumTopics(); ++k) {
IntPairWritable kw = new IntPairWritable(k, LDADriver.TOPIC_SUM_KEY);
v.set(logTotals[k]);
assert !Double.isNaN(v.get());
@@ -83,7 +83,7 @@ public class LDAMapper extends Mapper<Wr
}
IntPairWritable llk = new IntPairWritable(LDADriver.LOG_LIKELIHOOD_KEY, LDADriver.LOG_LIKELIHOOD_KEY);
// Output log-likelihoods.
- v.set(doc.logLikelihood);
+ v.set(doc.getLogLikelihood());
context.write(llk, v);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAState.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAState.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAState.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAState.java Fri May 28 20:03:36 2010
@@ -19,12 +19,13 @@ package org.apache.mahout.clustering.lda
import org.apache.mahout.math.Matrix;
public class LDAState {
- public final int numTopics;
- public final int numWords;
- public final double topicSmoothing;
+
+ private final int numTopics;
+ private final int numWords;
+ private final double topicSmoothing;
private final Matrix topicWordProbabilities; // log p(w|t) for topic=1..nTopics
private final double[] logTotals; // log \sum p(w|t) for topic=1..nTopics
- public final double logLikelihood; // log \sum p(w|t) for topic=1..nTopics
+ private final double logLikelihood; // log \sum p(w|t) for topic=1..nTopics
public LDAState(int numTopics,
int numWords,
@@ -44,4 +45,21 @@ public class LDAState {
double logProb = topicWordProbabilities.getQuick(topic, word);
return logProb == Double.NEGATIVE_INFINITY ? -100.0 : logProb - logTotals[topic];
}
+
+ public int getNumTopics() {
+ return numTopics;
+ }
+
+ public int getNumWords() {
+ return numWords;
+ }
+
+ public double getTopicSmoothing() {
+ return topicSmoothing;
+ }
+
+ public double getLogLikelihood() {
+ return logLikelihood;
+ }
+
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java Fri May 28 20:03:36 2010
@@ -43,10 +43,9 @@ public class MeanShiftCanopy extends Clu
// TODO: this is problematic, but how else to encode membership?
private IntArrayList boundPoints = new IntArrayList();
- private boolean converged = false;
+ private boolean converged;
public MeanShiftCanopy() {
- super();
}
/** Create a new Canopy with the given canopyId */
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java?rev=949271&r1=949270&r2=949271&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java Fri May 28 20:03:36 2010
@@ -18,7 +18,6 @@
package org.apache.mahout.clustering.meanshift;
import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.List;
@@ -44,15 +43,17 @@ public class MeanShiftCanopyClusterMappe
private List<MeanShiftCanopy> canopies;
@Override
- public void map(WritableComparable<?> key, MeanShiftCanopy canopy, OutputCollector<IntWritable, WeightedVectorWritable> output,
- Reporter reporter) throws IOException {
+ public void map(WritableComparable<?> key,
+ MeanShiftCanopy canopy,
+ OutputCollector<IntWritable, WeightedVectorWritable> output,
+ Reporter reporter) throws IOException {
// canopies use canopyIds assigned when input vectors are processed as vectorIds too
int vectorId = canopy.getId();
for (MeanShiftCanopy msc : canopies) {
for (int containedId : msc.getBoundPoints().toList()) {
if (vectorId == containedId) {
- // System.out.println(msc.getId() + ": v" + vectorId + "=" + ClusterBase.formatVector(vector.getCenter(), null));
- output.collect(new IntWritable(msc.getId()), new WeightedVectorWritable(1, new VectorWritable(canopy.getCenter())));
+ output.collect(new IntWritable(msc.getId()),
+ new WeightedVectorWritable(1, new VectorWritable(canopy.getCenter())));
}
}
}
@@ -67,15 +68,10 @@ public class MeanShiftCanopyClusterMappe
throw new IllegalStateException(e);
} catch (IllegalArgumentException e) {
throw new IllegalStateException(e);
- } catch (NoSuchMethodException e) {
- throw new IllegalStateException(e);
- } catch (InvocationTargetException e) {
- throw new IllegalStateException(e);
}
}
- public static List<MeanShiftCanopy> getCanopies(JobConf job) throws SecurityException, IllegalArgumentException,
- NoSuchMethodException, InvocationTargetException {
+ public static List<MeanShiftCanopy> getCanopies(JobConf job) throws SecurityException, IllegalArgumentException {
String statePath = job.get(MeanShiftCanopyDriver.STATE_IN_KEY);
List<MeanShiftCanopy> canopies = new ArrayList<MeanShiftCanopy>();
try {