You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by je...@apache.org on 2010/08/18 23:47:32 UTC
svn commit: r986960 [1/2] - in /mahout/trunk:
core/src/main/java/org/apache/mahout/clustering/
core/src/main/java/org/apache/mahout/clustering/canopy/
core/src/main/java/org/apache/mahout/clustering/dirichlet/
core/src/main/java/org/apache/mahout/clust...
Author: jeastman
Date: Wed Aug 18 21:47:30 2010
New Revision: 986960
URL: http://svn.apache.org/viewvc?rev=986960&view=rev
Log:
MAHOUT-479: Refactored Model and Cluster hierarchies to include DistanceMeasure in cluster state. All unit tests run
Added:
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/DistanceMeasureCluster.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/VectorModelClassifier.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/GaussianCluster.java
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/AbstractCluster.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusterer.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalDistribution.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1ModelDistribution.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/Model.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModelDistribution.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalDistribution.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalModel.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java
mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyDriver.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/TestClusterInterface.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/canopy/TestCanopyCreation.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/dirichlet/TestMapReduce.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestKmeansClustering.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/kmeans/TestRandomSeedGenerator.java
mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplayCanopy.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplayDirichlet.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplayFuzzyKMeans.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplayKMeans.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/display/DisplayMeanShift.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/canopy/Job.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/fuzzykmeans/Job.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/kmeans/Job.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/meanshift/InputMapper.java
mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/meanshift/Job.java
mahout/trunk/utils/src/main/java/org/apache/mahout/clustering/cdbw/CDbwDriver.java
mahout/trunk/utils/src/test/java/org/apache/mahout/clustering/TestClusterDumper.java
mahout/trunk/utils/src/test/java/org/apache/mahout/clustering/cdbw/TestCDbwEvaluator.java
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/AbstractCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/AbstractCluster.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/AbstractCluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/AbstractCluster.java Wed Aug 18 21:47:30 2010
@@ -8,8 +8,10 @@ import java.util.Iterator;
import java.util.Locale;
import org.apache.hadoop.io.Writable;
+import org.apache.mahout.clustering.dirichlet.models.Model;
import org.apache.mahout.math.JsonVectorAdapter;
import org.apache.mahout.math.NamedVector;
+import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.apache.mahout.math.function.SquareRootFunction;
@@ -18,7 +20,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-public abstract class AbstractCluster implements Writable, Cluster {
+public abstract class AbstractCluster implements Cluster, Model<VectorWritable> {
private static final Type VECTOR_TYPE = new TypeToken<Vector>() {
}.getType();
@@ -32,6 +34,23 @@ public abstract class AbstractCluster im
private Vector radius;
+ public AbstractCluster() {
+ }
+
+ public AbstractCluster(Vector point, int id2) {
+ this.numPoints = 0;
+ this.center = new RandomAccessSparseVector(point);
+ this.radius = point.like();
+ this.id = id2;
+ }
+
+ public AbstractCluster(Vector center2, Vector radius2, int id2) {
+ this.numPoints = 0;
+ this.center = new RandomAccessSparseVector(center2);
+ this.radius = new RandomAccessSparseVector(radius2);
+ this.id = id2;
+ }
+
/**
* @param id the id to set
*/
@@ -67,6 +86,8 @@ public abstract class AbstractCluster im
private transient Vector s2;
+ protected static final double SQRT2PI = Math.sqrt(2.0 * Math.PI);
+
/**
* @return the s0
*/
@@ -88,23 +109,6 @@ public abstract class AbstractCluster im
return s2;
}
- public void computeParameters() {
- if (s0 == 0) {
- return;
- }
- numPoints = (int) s0;
- center = s1.divide(s0);
- // compute the component stds
- if (s0 > 1) {
- radius = s2.times(s0).minus(s1.times(s1)).assign(new SquareRootFunction()).divide(s0);
- } else {
- radius.assign(Double.MIN_NORMAL);
- }
- s0 = 0;
- s1 = null;
- s2 = null;
- }
-
public void observe(ClusterObservations observations) {
s0 += observations.getS0();
if (s1 == null) {
@@ -119,8 +123,12 @@ public abstract class AbstractCluster im
}
}
- public ClusterObservations getObservations() {
- return new ClusterObservations(s0, s1, s2);
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.dirichlet.models.Model#observe(java.lang.Object)
+ */
+ @Override
+ public void observe(VectorWritable x) {
+ observe(x.get());
}
public void observe(Vector x, double weight) {
@@ -144,6 +152,32 @@ public abstract class AbstractCluster im
}
@Override
+ public int getNumPoints() {
+ return numPoints;
+ }
+
+ public ClusterObservations getObservations() {
+ return new ClusterObservations(s0, s1, s2);
+ }
+
+ public void computeParameters() {
+ if (s0 == 0) {
+ return;
+ }
+ numPoints = (int) s0;
+ center = s1.divide(s0);
+ // compute the component stds
+ if (s0 > 1) {
+ radius = s2.times(s0).minus(s1.times(s1)).assign(new SquareRootFunction()).divide(s0);
+ } else {
+ radius.assign(Double.MIN_NORMAL);
+ }
+ s0 = 0;
+ s1 = null;
+ s2 = null;
+ }
+
+ @Override
public void readFields(DataInput in) throws IOException {
this.id = in.readInt();
this.numPoints = in.readInt();
@@ -198,11 +232,6 @@ public abstract class AbstractCluster im
}
@Override
- public int getNumPoints() {
- return numPoints;
- }
-
- @Override
public Vector getRadius() {
return radius;
}
@@ -264,4 +293,12 @@ public abstract class AbstractCluster im
buf.append(']');
return buf.toString();
}
+
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.dirichlet.models.Model#count()
+ */
+ @Override
+ public int count() {
+ return getNumPoints();
+ }
}
Added: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/DistanceMeasureCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/DistanceMeasureCluster.java?rev=986960&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/DistanceMeasureCluster.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/DistanceMeasureCluster.java Wed Aug 18 21:47:30 2010
@@ -0,0 +1,81 @@
+package org.apache.mahout.clustering;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.mahout.clustering.dirichlet.models.Model;
+import org.apache.mahout.common.distance.DistanceMeasure;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.VectorWritable;
+
+public abstract class DistanceMeasureCluster extends AbstractCluster {
+
+ protected DistanceMeasure measure;
+
+ public DistanceMeasureCluster(Vector point, int id, DistanceMeasure measure2) {
+ super(point,id);
+ this.measure = measure2;
+ }
+
+ public DistanceMeasureCluster() {
+ super();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.AbstractCluster#readFields(java.io.DataInput)
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ String dm = in.readUTF();
+ try {
+ ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+ this.measure = (DistanceMeasure) ((Class<?>) ccl.loadClass(dm)).newInstance();
+ } catch (InstantiationException e) {
+ throw new IllegalStateException(e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(e);
+ } catch (ClassNotFoundException e) {
+ throw new IllegalStateException(e);
+ }
+ super.readFields(in);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.AbstractCluster#write(java.io.DataOutput)
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeUTF(measure.getClass().getName());
+ super.write(out);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.AbstractCluster#pdf(org.apache.mahout.math.VectorWritable)
+ */
+ @Override
+ public double pdf(VectorWritable vw) {
+ return 1 / measure.distance(getCenter(), vw.get());
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.dirichlet.models.Model#sampleFromPosterior()
+ */
+ @Override
+ public Model<VectorWritable> sampleFromPosterior() {
+ // TODO: fix this
+ return this;
+ }
+
+ public DistanceMeasure getMeasure() {
+ return measure;
+ }
+
+ /**
+ * @param measure the measure to set
+ */
+ public void setMeasure(DistanceMeasure measure) {
+ this.measure = measure;
+ }
+
+}
Added: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/VectorModelClassifier.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/VectorModelClassifier.java?rev=986960&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/VectorModelClassifier.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/VectorModelClassifier.java Wed Aug 18 21:47:30 2010
@@ -0,0 +1,45 @@
+package org.apache.mahout.clustering;
+
+import java.util.List;
+
+import org.apache.mahout.classifier.AbstractVectorClassifier;
+import org.apache.mahout.clustering.dirichlet.models.Model;
+import org.apache.mahout.math.DenseVector;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.VectorWritable;
+import org.apache.mahout.math.function.TimesFunction;
+
+public class VectorModelClassifier extends AbstractVectorClassifier {
+
+ List<Model<VectorWritable>> models;
+
+ public VectorModelClassifier(List<Model<VectorWritable>> models) {
+ super();
+ this.models = models;
+ }
+
+ @Override
+ public Vector classify(Vector instance) {
+ Vector pdfs = new DenseVector(models.size());
+ int i = 0;
+ for (Model<VectorWritable> model : models) {
+ pdfs.set(i++, model.pdf(new VectorWritable(instance)));
+ }
+ return pdfs.assign(new TimesFunction(), 1.0 / pdfs.zSum());
+ }
+
+ @Override
+ public double classifyScalar(Vector instance) {
+ if (models.size() == 2) {
+ double pdf0 = models.get(0).pdf(new VectorWritable(instance));
+ double pdf1 = models.get(1).pdf(new VectorWritable(instance));
+ return pdf0 / (pdf0 + pdf1);
+ }
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public int numCategories() {
+ return models.size();
+ }
+}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/Canopy.java Wed Aug 18 21:47:30 2010
@@ -20,9 +20,8 @@ package org.apache.mahout.clustering.can
import java.io.DataInput;
import java.io.IOException;
-import org.apache.mahout.clustering.AbstractCluster;
-import org.apache.mahout.math.AbstractVector;
-import org.apache.mahout.math.RandomAccessSparseVector;
+import org.apache.mahout.clustering.DistanceMeasureCluster;
+import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.math.Vector;
/**
@@ -30,7 +29,7 @@ import org.apache.mahout.math.Vector;
* to the application of some distance metric, and a point total which is the sum of all the points and is
* used to compute the centroid when needed.
*/
-public class Canopy extends AbstractCluster {
+public class Canopy extends DistanceMeasureCluster {
/** Used for deserializaztion as a writable */
public Canopy() { }
@@ -38,16 +37,12 @@ public class Canopy extends AbstractClus
/**
* Create a new Canopy containing the given point and canopyId
*
- * @param center
- * a point in vector space
- * @param canopyId
- * an int identifying the canopy local to this process only
+ * @param center a point in vector space
+ * @param canopyId an int identifying the canopy local to this process only
+ * @param measure a DistanceMeasure to use
*/
- public Canopy(Vector center, int canopyId) {
- this.setId(canopyId);
- this.setNumPoints(0);
- this.setCenter(new RandomAccessSparseVector(center));
- this.setRadius(center.like());
+ public Canopy(Vector center, int canopyId, DistanceMeasure measure) {
+ super(center, canopyId, measure);
observe(center);
}
@@ -56,32 +51,8 @@ public class Canopy extends AbstractClus
super.readFields(in);
}
- /** Format the canopy for output */
- public static String formatCanopy(Canopy canopy) {
- return "C" + canopy.getId() + ": " + canopy.computeCentroid().asFormatString();
- }
-
public String asFormatString() {
- return formatCanopy(this);
- }
-
- /**
- * Decodes and returns a Canopy from the formattedString
- *
- * @param formattedString
- * a String prouced by formatCanopy
- * @return a new Canopy
- */
- public static Canopy decodeCanopy(String formattedString) {
- int beginIndex = formattedString.indexOf('{');
- String id = formattedString.substring(0, beginIndex);
- String centroid = formattedString.substring(beginIndex);
- if (id.charAt(0) == 'C') {
- int canopyId = Integer.parseInt(formattedString.substring(1, beginIndex - 2));
- Vector canopyCentroid = AbstractVector.decodeVector(centroid);
- return new Canopy(canopyCentroid, canopyId);
- }
- return null;
+ return "C" + this.getId() + ": " + this.computeCentroid().asFormatString();
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusterer.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyClusterer.java Wed Aug 18 21:47:30 2010
@@ -120,7 +120,7 @@ public class CanopyClusterer {
}
if (!pointStronglyBound) {
log.info("Created new Canopy:" + nextCanopyId + " at center:" + AbstractCluster.formatVector(point, null));
- canopies.add(new Canopy(point, nextCanopyId++));
+ canopies.add(new Canopy(point, nextCanopyId++, measure));
}
}
@@ -195,7 +195,7 @@ public class CanopyClusterer {
Iterator<Vector> ptIter = points.iterator();
Vector p1 = ptIter.next();
ptIter.remove();
- Canopy canopy = new Canopy(p1, nextCanopyId++);
+ Canopy canopy = new Canopy(p1, nextCanopyId++, measure);
canopies.add(canopy);
while (ptIter.hasNext()) {
Vector p2 = ptIter.next();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyDriver.java Wed Aug 18 21:47:30 2010
@@ -64,8 +64,8 @@ public class CanopyDriver extends Abstra
* the input pathname String
* @param output
* the output pathname String
- * @param measureClassName
- * the DistanceMeasure class name
+ * @param measure
+ * the DistanceMeasure
* @param t1
* the T1 distance threshold
* @param t2
@@ -80,13 +80,13 @@ public class CanopyDriver extends Abstra
*/
public static void runJob(Path input,
Path output,
- String measureClassName,
+ DistanceMeasure measure,
double t1,
double t2,
boolean runClustering,
boolean runSequential) throws IOException, InterruptedException, ClassNotFoundException,
InstantiationException, IllegalAccessException {
- new CanopyDriver().job(input, output, measureClassName, t1, t2, runClustering, runSequential);
+ new CanopyDriver().job(input, output, measure, t1, t2, runClustering, runSequential);
}
@Override
@@ -116,8 +116,10 @@ public class CanopyDriver extends Abstra
double t2 = Double.parseDouble(getOption(DefaultOptionCreator.T2_OPTION));
boolean runClustering = hasOption(DefaultOptionCreator.CLUSTERING_OPTION);
boolean runSequential = (getOption(DefaultOptionCreator.METHOD_OPTION).equalsIgnoreCase(DefaultOptionCreator.SEQUENTIAL_METHOD));
+ ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+ DistanceMeasure measure = (DistanceMeasure) ccl.loadClass(measureClass).newInstance();
- job(input, output, measureClass, t1, t2, runClustering, runSequential);
+ job(input, output, measure, t1, t2, runClustering, runSequential);
return 0;
}
@@ -139,15 +141,15 @@ public class CanopyDriver extends Abstra
*/
public void job(Path input,
Path output,
- String measureClassName,
+ DistanceMeasure measure,
double t1,
double t2,
boolean runClustering,
boolean runSequential) throws IOException, InterruptedException, ClassNotFoundException, InstantiationException,
IllegalAccessException {
- Path clustersOut = buildClusters(input, output, measureClassName, t1, t2, runSequential);
+ Path clustersOut = buildClusters(input, output, measure, t1, t2, runSequential);
if (runClustering) {
- clusterData(input, clustersOut, output, measureClassName, t1, t2, runSequential);
+ clusterData(input, clustersOut, output, measure, t1, t2, runSequential);
}
}
@@ -157,7 +159,7 @@ public class CanopyDriver extends Abstra
*
* @param input the Path to the directory containing input vectors
* @param output the Path for all output directories
- * @param measureClassName the String class name of the DistanceMeasure
+ * @param measure the DistanceMeasure
* @param t1 the double T1 distance metric
* @param t2 the double T2 distance metric
* @param runSequential a boolean indicates to run the sequential (reference) algorithm
@@ -168,13 +170,13 @@ public class CanopyDriver extends Abstra
* @throws IllegalAccessException
* @throws InstantiationException
*/
- public Path buildClusters(Path input, Path output, String measureClassName, double t1, double t2, boolean runSequential)
+ public Path buildClusters(Path input, Path output, DistanceMeasure measure, double t1, double t2, boolean runSequential)
throws IOException, InterruptedException, ClassNotFoundException, InstantiationException, IllegalAccessException {
- log.info("Input: {} Out: {} " + "Measure: {} t1: {} t2: {}", new Object[] { input, output, measureClassName, t1, t2 });
+ log.info("Input: {} Out: {} " + "Measure: {} t1: {} t2: {}", new Object[] { input, output, measure, t1, t2 });
if (runSequential) {
- return buildClustersSeq(input, output, measureClassName, t1, t2);
+ return buildClustersSeq(input, output, measure, t1, t2);
} else {
- return buildClustersMR(input, output, measureClassName, t1, t2);
+ return buildClustersMR(input, output, measure, t1, t2);
}
}
@@ -184,15 +186,13 @@ public class CanopyDriver extends Abstra
*
* @param input the Path to the directory containing input vectors
* @param output the Path for all output directories
- * @param measureClassName the String class name of the DistanceMeasure
+ * @param measure the DistanceMeasure
* @param t1 the double T1 distance metric
* @param t2 the double T2 distance metric
* @return the canopy output directory Path
*/
- private Path buildClustersSeq(Path input, Path output, String measureClassName, double t1, double t2)
+ private Path buildClustersSeq(Path input, Path output, DistanceMeasure measure, double t1, double t2)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IOException {
- ClassLoader ccl = Thread.currentThread().getContextClassLoader();
- DistanceMeasure measure = (DistanceMeasure) ccl.loadClass(measureClassName).newInstance();
CanopyClusterer clusterer = new CanopyClusterer(measure, t1, t2);
List<Canopy> canopies = new ArrayList<Canopy>();
Configuration conf = new Configuration();
@@ -233,15 +233,15 @@ public class CanopyDriver extends Abstra
*
* @param input the Path to the directory containing input vectors
* @param output the Path for all output directories
- * @param measureClassName the String class name of the DistanceMeasure
+ * @param measure the DistanceMeasure
* @param t1 the double T1 distance metric
* @param t2 the double T2 distance metric
* @return the canopy output directory Path
*/
- private Path buildClustersMR(Path input, Path output, String measureClassName, double t1, double t2)
- throws IOException, InterruptedException, ClassNotFoundException {
+ private Path buildClustersMR(Path input, Path output, DistanceMeasure measure, double t1, double t2) throws IOException,
+ InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
- conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, measureClassName);
+ conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, measure.getClass().getName());
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(t1));
conf.set(CanopyConfigKeys.T2_KEY, String.valueOf(t2));
@@ -270,7 +270,7 @@ public class CanopyDriver extends Abstra
* @param points the input points directory pathname String
* @param canopies the input canopies directory pathname String
* @param output the output directory pathname String
- * @param measureClassName the DistanceMeasure class name
+ * @param measure the DistanceMeasure
* @param t1 the T1 distance threshold
* @param t2 the T2 distance threshold
* @param runSequential a boolean indicates to run the sequential (reference) algorithm
@@ -282,27 +282,20 @@ public class CanopyDriver extends Abstra
public void clusterData(Path points,
Path canopies,
Path output,
- String measureClassName,
+ DistanceMeasure measure,
double t1,
double t2,
boolean runSequential) throws IOException, InterruptedException, ClassNotFoundException,
InstantiationException, IllegalAccessException {
if (runSequential) {
- clusterDataSeq(points, canopies, output, measureClassName, t1, t2);
+ clusterDataSeq(points, canopies, output, measure, t1, t2);
} else {
- clusterDataMR(points, canopies, output, measureClassName, t1, t2);
+ clusterDataMR(points, canopies, output, measure, t1, t2);
}
}
- private void clusterDataSeq(Path points,
- Path canopies,
- Path output,
- String measureClassName,
- double t1,
- double t2)
+ private void clusterDataSeq(Path points, Path canopies, Path output, DistanceMeasure measure, double t1, double t2)
throws InstantiationException, IllegalAccessException, ClassNotFoundException, IOException {
- ClassLoader ccl = Thread.currentThread().getContextClassLoader();
- DistanceMeasure measure = (DistanceMeasure) ccl.loadClass(measureClassName).newInstance();
CanopyClusterer clusterer = new CanopyClusterer(measure, t1, t2);
List<Canopy> clusters = new ArrayList<Canopy>();
@@ -349,15 +342,10 @@ public class CanopyDriver extends Abstra
}
}
- private void clusterDataMR(Path points,
- Path canopies,
- Path output,
- String measureClassName,
- double t1,
- double t2)
+ private void clusterDataMR(Path points, Path canopies, Path output, DistanceMeasure measure, double t1, double t2)
throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
- conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, measureClassName);
+ conf.set(CanopyConfigKeys.DISTANCE_MEASURE_KEY, measure.getClass().getName());
conf.set(CanopyConfigKeys.T1_KEY, String.valueOf(t1));
conf.set(CanopyConfigKeys.T2_KEY, String.valueOf(t2));
conf.set(CanopyConfigKeys.CANOPY_PATH_KEY, canopies.toString());
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletCluster.java Wed Aug 18 21:47:30 2010
@@ -76,6 +76,14 @@ public class DirichletCluster<O> impleme
writeModel(out, model);
}
+ /************* Methods required by Cluster *****************/
+
+ /** Writes a typed Model instance to the output stream */
+ public static void writeModel(DataOutput out, Model<?> model) throws IOException {
+ out.writeUTF(model.getClass().getName());
+ model.write(out);
+ }
+
/** Reads a typed Model instance from the input stream */
public static <O> Model<O> readModel(DataInput in) throws IOException {
String modelClassName = in.readUTF();
@@ -93,15 +101,11 @@ public class DirichletCluster<O> impleme
return model;
}
- /** Writes a typed Model instance to the output stream */
- public static void writeModel(DataOutput out, Model<?> model) throws IOException {
- out.writeUTF(model.getClass().getName());
- model.write(out);
- }
-
+ /************* Methods required by Cluster *****************/
+
@Override
public String asFormatString(String[] bindings) {
- return "C-" + model.getId() + ": " + model.asFormatString(bindings);
+ return "C-" + ((Cluster)model).getId() + ": " + ((Cluster)model).asFormatString(bindings);
}
@Override
@@ -114,22 +118,22 @@ public class DirichletCluster<O> impleme
@Override
public int getId() {
- return model.getId();
+ return ((Cluster)model).getId();
}
@Override
public Vector getCenter() {
- return model.getCenter();
+ return ((Cluster)model).getCenter();
}
@Override
public int getNumPoints() {
- return model.getNumPoints();
+ return ((Cluster)model).getNumPoints();
}
@Override
public Vector getRadius() {
- return model.getRadius();
+ return ((Cluster) model).getRadius();
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalDistribution.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalDistribution.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalDistribution.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalDistribution.java Wed Aug 18 21:47:30 2010
@@ -58,7 +58,7 @@ public class AsymmetricSampledNormalDist
Model<VectorWritable>[] result = new AsymmetricSampledNormalModel[posterior.length];
for (int i = 0; i < posterior.length; i++) {
AsymmetricSampledNormalModel m = (AsymmetricSampledNormalModel) posterior[i];
- result[i] = m.sample();
+ result[i] = m.sampleFromPosterior();
}
return result;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/AsymmetricSampledNormalModel.java Wed Aug 18 21:47:30 2010
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.lang.reflect.Type;
import org.apache.mahout.clustering.AbstractCluster;
+import org.apache.mahout.clustering.Cluster;
import org.apache.mahout.clustering.dirichlet.JsonModelAdapter;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -32,7 +33,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-public class AsymmetricSampledNormalModel implements Model<VectorWritable> {
+public class AsymmetricSampledNormalModel implements Model<VectorWritable>, Cluster {
private static final double SQRT2PI = Math.sqrt(2.0 * Math.PI);
private static final Type MODEL_TYPE = new TypeToken<Model<Vector>>() { }.getType();
@@ -76,7 +77,7 @@ public class AsymmetricSampledNormalMode
*
* @return an AsymmetricSampledNormalModel
*/
- AsymmetricSampledNormalModel sample() {
+ public AsymmetricSampledNormalModel sampleFromPosterior() {
return new AsymmetricSampledNormalModel(id, mean, stdDev);
}
Added: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/GaussianCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/GaussianCluster.java?rev=986960&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/GaussianCluster.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/GaussianCluster.java Wed Aug 18 21:47:30 2010
@@ -0,0 +1,61 @@
+package org.apache.mahout.clustering.dirichlet.models;
+
+import org.apache.mahout.clustering.AbstractCluster;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.VectorWritable;
+
+public class GaussianCluster extends AbstractCluster {
+
+ public GaussianCluster() {
+ super();
+ }
+
+ public GaussianCluster(Vector point, int id2) {
+ super(point, id2);
+ }
+
+ public GaussianCluster(Vector center, Vector radius, int id) {
+ super(center, radius, id);
+ }
+
+ @Override
+ public String getIdentifier() {
+ return "GC:" + getId();
+ }
+
+ @Override
+ public Model<VectorWritable> sampleFromPosterior() {
+ return new GaussianCluster(getCenter(), getRadius(), getId());
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.mahout.clustering.dirichlet.models.Model#pdf(java.lang.Object)
+ */
+ @Override
+ public double pdf(VectorWritable vw) {
+ Vector x = vw.get();
+ // return the product of the component pdfs
+ // TODO: is this reasonable? correct?
+ double pdf = pdf(x, getRadius().get(0));
+ for (int i = 1; i < x.size(); i++) {
+ pdf *= pdf(x, getRadius().get(i));
+ }
+ return pdf;
+ }
+
+ /**
+ * Calculate a pdf using the supplied sample and stdDev
+ *
+ * @param x
+ * a Vector sample
+ * @param sd
+ * a double std deviation
+ */
+ private double pdf(Vector x, double sd) {
+ double sd2 = sd * sd;
+ double exp = -(x.dot(x) - 2 * x.dot(getCenter()) + getCenter().dot(getCenter())) / (2 * sd2);
+ double ex = Math.exp(exp);
+ return ex / (sd * SQRT2PI);
+ }
+
+}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1Model.java Wed Aug 18 21:47:30 2010
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.lang.reflect.Type;
import org.apache.mahout.clustering.AbstractCluster;
+import org.apache.mahout.clustering.Cluster;
import org.apache.mahout.clustering.dirichlet.JsonModelAdapter;
import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.common.distance.ManhattanDistanceMeasure;
@@ -32,7 +33,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-public class L1Model implements Model<VectorWritable> {
+public class L1Model implements Model<VectorWritable>, Cluster {
private static final DistanceMeasure MEASURE = new ManhattanDistanceMeasure();
private static final Type MODEL_TYPE = new TypeToken<Model<Vector>>() {}.getType();
@@ -92,7 +93,7 @@ public class L1Model implements Model<Ve
VectorWritable.writeVector(out, coefficients);
}
- public L1Model sample() {
+ public L1Model sampleFromPosterior() {
return new L1Model(id, coefficients.clone());
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1ModelDistribution.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1ModelDistribution.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1ModelDistribution.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/L1ModelDistribution.java Wed Aug 18 21:47:30 2010
@@ -47,7 +47,7 @@ public class L1ModelDistribution extends
public Model<VectorWritable>[] sampleFromPosterior(Model<VectorWritable>[] posterior) {
Model<VectorWritable>[] result = new L1Model[posterior.length];
for (int i = 0; i < posterior.length; i++) {
- result[i] = ((L1Model) posterior[i]).sample();
+ result[i] = ((L1Model) posterior[i]).sampleFromPosterior();
}
return result;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/Model.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/Model.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/Model.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/Model.java Wed Aug 18 21:47:30 2010
@@ -18,15 +18,24 @@
package org.apache.mahout.clustering.dirichlet.models;
import org.apache.hadoop.io.Writable;
-import org.apache.mahout.clustering.Cluster;
+import org.apache.mahout.math.VectorWritable;
/**
* A model is a probability distribution over observed data points and allows the probability of any data
- * point to be computed.
+ * point to be computed. All Models have a persistent representation and extend WritablesampleFromPosterior(Model<VectorWritable>[])
*/
-public interface Model<O> extends Writable, Cluster {
+public interface Model<O> extends Writable {
/**
+ * Return the probability that the observation is described by this model
+ *
+ * @param x
+ * an Observation from the posterior
+ * @return the probability that x is in the receiver
+ */
+ double pdf(O x);
+
+ /**
* Observe the given observation, retaining information about it
*
* @param x
@@ -41,18 +50,14 @@ public interface Model<O> extends Writab
void computeParameters();
/**
- * Return the probability that the observation is described by this model
- *
- * @param x
- * an Observation from the posterior
- * @return the probability that x is in the receiver
- */
- double pdf(O x);
-
- /**
* Return the number of observations that have been observed by this model
*
* @return an int
*/
int count();
+
+ /**
+ * @return a sample of my posterior model
+ */
+ public Model<VectorWritable> sampleFromPosterior();
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModel.java Wed Aug 18 21:47:30 2010
@@ -24,6 +24,7 @@ import java.lang.reflect.Type;
import java.util.Locale;
import org.apache.mahout.clustering.AbstractCluster;
+import org.apache.mahout.clustering.Cluster;
import org.apache.mahout.clustering.dirichlet.JsonModelAdapter;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
@@ -33,7 +34,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
-public class NormalModel implements Model<VectorWritable> {
+public class NormalModel implements Model<VectorWritable>, Cluster {
private static final double SQRT2PI = Math.sqrt(2.0 * Math.PI);
@@ -81,7 +82,7 @@ public class NormalModel implements Mode
*
* @return an NormalModel
*/
- public NormalModel sample() {
+ public NormalModel sampleFromPosterior() {
return new NormalModel(id, mean, stdDev);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModelDistribution.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModelDistribution.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModelDistribution.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/NormalModelDistribution.java Wed Aug 18 21:47:30 2010
@@ -48,7 +48,7 @@ public class NormalModelDistribution ext
Model<VectorWritable>[] result = new NormalModel[posterior.length];
for (int i = 0; i < posterior.length; i++) {
NormalModel m = (NormalModel) posterior[i];
- result[i] = m.sample();
+ result[i] = m.sampleFromPosterior();
}
return result;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalDistribution.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalDistribution.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalDistribution.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalDistribution.java Wed Aug 18 21:47:30 2010
@@ -56,7 +56,7 @@ public class SampledNormalDistribution e
Model<VectorWritable>[] result = new SampledNormalModel[posterior.length];
for (int i = 0; i < posterior.length; i++) {
SampledNormalModel m = (SampledNormalModel) posterior[i];
- result[i] = m.sample();
+ result[i] = m.sampleFromPosterior();
}
return result;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalModel.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/models/SampledNormalModel.java Wed Aug 18 21:47:30 2010
@@ -42,7 +42,7 @@ public class SampledNormalModel extends
* @return an SampledNormalModel
*/
@Override
- public NormalModel sample() {
+ public NormalModel sampleFromPosterior() {
return new SampledNormalModel(getId(), getMean(), getStdDev());
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansClusterer.java Wed Aug 18 21:47:30 2010
@@ -96,7 +96,7 @@ public class FuzzyKMeansClusterer {
List<SoftCluster> next = new ArrayList<SoftCluster>();
List<SoftCluster> cs = clustersList.get(iteration++);
for (SoftCluster c : cs) {
- next.add(new SoftCluster(c.getCenter(), c.getId()));
+ next.add(new SoftCluster(c.getCenter(), c.getId(), measure));
}
clustersList.add(next);
converged = runFuzzyKMeansIteration(points, clustersList.get(iteration), clusterer);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java Wed Aug 18 21:47:30 2010
@@ -72,8 +72,8 @@ public class FuzzyKMeansDriver extends A
* the directory pathname for initial & computed clusters
* @param output
* the directory pathname for output points
- * @param measureClass
- * the classname of the DistanceMeasure
+ * @param measure
+ * the DistanceMeasure
* @param convergenceDelta
* the convergence delta value
* @param maxIterations
@@ -97,7 +97,7 @@ public class FuzzyKMeansDriver extends A
public static void runJob(Path input,
Path clustersIn,
Path output,
- String measureClass,
+ DistanceMeasure measure,
double convergenceDelta,
int maxIterations,
int numReduceTasks,
@@ -111,7 +111,7 @@ public class FuzzyKMeansDriver extends A
new FuzzyKMeansDriver().job(input,
clustersIn,
output,
- measureClass,
+ measure,
convergenceDelta,
maxIterations,
numReduceTasks,
@@ -167,16 +167,19 @@ public class FuzzyKMeansDriver extends A
}
boolean emitMostLikely = Boolean.parseBoolean(getOption(DefaultOptionCreator.EMIT_MOST_LIKELY_OPTION));
double threshold = Double.parseDouble(getOption(DefaultOptionCreator.THRESHOLD_OPTION));
+ ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+ DistanceMeasure measure = (DistanceMeasure) ((Class<?>) ccl.loadClass(measureClass)).newInstance();
+
if (hasOption(DefaultOptionCreator.NUM_CLUSTERS_OPTION)) {
clusters = RandomSeedGenerator.buildRandom(input, clusters, Integer.parseInt(parseArguments(args)
- .get(DefaultOptionCreator.NUM_CLUSTERS_OPTION)));
+ .get(DefaultOptionCreator.NUM_CLUSTERS_OPTION)), measure);
}
boolean runClustering = hasOption(DefaultOptionCreator.CLUSTERING_OPTION);
boolean runSequential = (getOption(DefaultOptionCreator.METHOD_OPTION).equalsIgnoreCase(DefaultOptionCreator.SEQUENTIAL_METHOD));
job(input,
clusters,
output,
- measureClass,
+ measure,
convergenceDelta,
maxIterations,
numReduceTasks,
@@ -297,7 +300,7 @@ public class FuzzyKMeansDriver extends A
public void job(Path input,
Path clustersIn,
Path output,
- String measureClass,
+ DistanceMeasure measure,
double convergenceDelta,
int maxIterations,
int numReduceTasks,
@@ -307,9 +310,6 @@ public class FuzzyKMeansDriver extends A
double threshold,
boolean runSequential) throws IOException, ClassNotFoundException, InterruptedException, InstantiationException,
IllegalAccessException {
- ClassLoader ccl = Thread.currentThread().getContextClassLoader();
- Class<?> cl = ccl.loadClass(measureClass);
- DistanceMeasure measure = (DistanceMeasure) cl.newInstance();
Path clustersOut = buildClusters(input,
clustersIn,
output,
@@ -377,13 +377,13 @@ public class FuzzyKMeansDriver extends A
}
private Path buildClustersSeq(Path input,
- Path clustersIn,
- Path output,
- DistanceMeasure measure,
- double convergenceDelta,
- int maxIterations,
- int numReduceTasks,
- float m) throws IOException, InstantiationException, IllegalAccessException {
+ Path clustersIn,
+ Path output,
+ DistanceMeasure measure,
+ double convergenceDelta,
+ int maxIterations,
+ int numReduceTasks,
+ float m) throws IOException, InstantiationException, IllegalAccessException {
FuzzyKMeansClusterer clusterer = new FuzzyKMeansClusterer(measure, convergenceDelta, m);
List<SoftCluster> clusters = new ArrayList<SoftCluster>();
@@ -421,8 +421,8 @@ public class FuzzyKMeansDriver extends A
try {
for (SoftCluster cluster : clusters) {
log.info("Writing Cluster:" + cluster.getId() + " center:" + AbstractCluster.formatVector(cluster.getCenter(), null)
- + " numPoints:" + cluster.getNumPoints() + " radius:" + AbstractCluster.formatVector(cluster.getRadius(), null) + " to: "
- + clustersOut.getName());
+ + " numPoints:" + cluster.getNumPoints() + " radius:" + AbstractCluster.formatVector(cluster.getRadius(), null)
+ + " to: " + clustersOut.getName());
writer.append(new Text(cluster.getIdentifier()), cluster);
}
} finally {
@@ -507,14 +507,14 @@ public class FuzzyKMeansDriver extends A
}
private void clusterDataSeq(Path input,
- Path clustersIn,
- Path output,
- DistanceMeasure measure,
- double convergenceDelta,
- float m,
- boolean emitMostLikely,
- double threshold)
- throws IOException, InterruptedException, InstantiationException, IllegalAccessException {
+ Path clustersIn,
+ Path output,
+ DistanceMeasure measure,
+ double convergenceDelta,
+ float m,
+ boolean emitMostLikely,
+ double threshold) throws IOException, InterruptedException, InstantiationException,
+ IllegalAccessException {
FuzzyKMeansClusterer clusterer = new FuzzyKMeansClusterer(measure, convergenceDelta, m);
List<SoftCluster> clusters = new ArrayList<SoftCluster>();
FuzzyKMeansUtil.configureWithClusterInfo(clustersIn, clusters);
@@ -546,14 +546,15 @@ public class FuzzyKMeansDriver extends A
}
}
+
private void clusterDataMR(Path input,
- Path clustersIn,
- Path output,
- DistanceMeasure measure,
- double convergenceDelta,
- float m,
- boolean emitMostLikely,
- double threshold) throws IOException, InterruptedException, ClassNotFoundException {
+ Path clustersIn,
+ Path output,
+ DistanceMeasure measure,
+ double convergenceDelta,
+ float m,
+ boolean emitMostLikely,
+ double threshold) throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set(FuzzyKMeansConfigKeys.CLUSTER_PATH_KEY, clustersIn.toString());
conf.set(FuzzyKMeansConfigKeys.DISTANCE_MEASURE_KEY, measure.getClass().getName());
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansUtil.java Wed Aug 18 21:47:30 2010
@@ -29,7 +29,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Writable;
-import org.apache.mahout.clustering.AbstractCluster;
import org.apache.mahout.clustering.canopy.Canopy;
import org.apache.mahout.clustering.kmeans.Cluster;
import org.slf4j.Logger;
@@ -85,10 +84,10 @@ final class FuzzyKMeansUtil {
throw new IllegalStateException(e);
}
if (valueClass.equals(Cluster.class)) {
- AbstractCluster value = new Cluster();
+ Cluster value = new Cluster();
while (reader.next(key, value)) {
// get the cluster info
- SoftCluster theCluster = new SoftCluster(value.getCenter(), value.getId());
+ SoftCluster theCluster = new SoftCluster(value.getCenter(), value.getId(), value.getMeasure());
clusters.add(theCluster);
value = new Cluster();
}
@@ -103,7 +102,7 @@ final class FuzzyKMeansUtil {
Canopy value = new Canopy();
while (reader.next(key, value)) {
// get the cluster info
- SoftCluster theCluster = new SoftCluster(value.getCenter(), value.getId());
+ SoftCluster theCluster = new SoftCluster(value.getCenter(), value.getId(), value.getMeasure());
clusters.add(theCluster);
value = new Canopy();
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java Wed Aug 18 21:47:30 2010
@@ -18,7 +18,7 @@
package org.apache.mahout.clustering.fuzzykmeans;
import org.apache.mahout.clustering.kmeans.Cluster;
-import org.apache.mahout.math.AbstractVector;
+import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.math.Vector;
public class SoftCluster extends Cluster {
@@ -30,49 +30,16 @@ public class SoftCluster extends Cluster
/**
* Construct a new SoftCluster with the given point as its center
*
- * @param center
- * the center point
+ * @param center the center point
+ * @param measure the DistanceMeasure
*/
- public SoftCluster(Vector center, int clusterId) {
- super(center, clusterId);
- }
-
- /**
- * Format the SoftCluster for output
- *
- * @param cluster
- * the Cluster
- */
- public static String formatCluster(SoftCluster cluster) {
- return cluster.getIdentifier() + ": " + cluster.computeCentroid().asFormatString();
- }
-
- /**
- * Decodes and returns a SoftCluster from the formattedString
- *
- * @param formattedString
- * a String produced by formatCluster
- */
- public static SoftCluster decodeCluster(String formattedString) {
- int beginIndex = formattedString.indexOf('{');
- String id = formattedString.substring(0, beginIndex);
- String center = formattedString.substring(beginIndex);
- char firstChar = id.charAt(0);
- boolean startsWithV = firstChar == 'V';
- if ((firstChar == 'C') || startsWithV) {
- int clusterId = Integer.parseInt(formattedString.substring(1, beginIndex - 2));
- Vector clusterCenter = AbstractVector.decodeVector(center);
-
- SoftCluster cluster = new SoftCluster(clusterCenter, clusterId);
- cluster.setConverged(startsWithV);
- return cluster;
- }
- return null;
+ public SoftCluster(Vector center, int clusterId, DistanceMeasure measure) {
+ super(center, clusterId, measure);
}
@Override
public String asFormatString() {
- return formatCluster(this);
+ return this.getIdentifier() + ": " + this.computeCentroid().asFormatString();
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/Cluster.java Wed Aug 18 21:47:30 2010
@@ -20,35 +20,29 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import org.apache.mahout.clustering.AbstractCluster;
+import org.apache.mahout.clustering.DistanceMeasureCluster;
import org.apache.mahout.common.distance.DistanceMeasure;
-import org.apache.mahout.math.AbstractVector;
-import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.Vector;
-public class Cluster extends AbstractCluster {
-
- /** Error message for unknown cluster format in output. */
- private static final String ERROR_UNKNOWN_CLUSTER_FORMAT = "Unknown cluster format:\n";
+public class Cluster extends DistanceMeasureCluster {
/** Has the centroid converged with the center? */
private boolean converged;
/** For (de)serialization as a Writable */
public Cluster() {
+ super();
}
/**
* Construct a new cluster with the given point as its center
*
- * @param center
- * the center point
+ * @param center the Vector center
+ * @param clusterId the int cluster id
+ * @param measure a DistanceMeasure
*/
- public Cluster(Vector center, int clusterId) {
- this.setId(clusterId);
- this.setNumPoints(0);
- this.setCenter(new RandomAccessSparseVector(center));
- this.setRadius(center.like());
+ public Cluster(Vector center, int clusterId, DistanceMeasure measure) {
+ super(center, clusterId, measure);
}
/**
@@ -66,36 +60,6 @@ public class Cluster extends AbstractClu
return formatCluster(this);
}
- /**
- * Decodes and returns a Cluster from the formattedString.
- *
- * @param formattedString
- * a String produced by formatCluster
- * @return a decoded Cluster, not null
- * @throws IllegalArgumentException
- * when the string is wrongly formatted
- */
- public static AbstractCluster decodeCluster(String formattedString) {
- int beginIndex = formattedString.indexOf('{');
- if (beginIndex <= 0) {
- throw new IllegalArgumentException(ERROR_UNKNOWN_CLUSTER_FORMAT + formattedString);
- }
- String id = formattedString.substring(0, beginIndex);
- String center = formattedString.substring(beginIndex);
- char firstChar = id.charAt(0);
- boolean startsWithV = firstChar == 'V';
- Cluster cluster;
- if ((firstChar == 'C') || startsWithV) {
- int clusterId = Integer.parseInt(formattedString.substring(1, beginIndex - 2));
- Vector clusterCenter = AbstractVector.decodeVector(center);
- cluster = new Cluster(clusterCenter, clusterId);
- cluster.setConverged(startsWithV);
- } else {
- throw new IllegalArgumentException(ERROR_UNKNOWN_CLUSTER_FORMAT + formattedString);
- }
- return cluster;
- }
-
@Override
public void write(DataOutput out) throws IOException {
super.write(out);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterer.java Wed Aug 18 21:47:30 2010
@@ -201,7 +201,7 @@ public class KMeansClusterer {
log.info("Reference Iteration: " + iteration);
List<Cluster> next = new ArrayList<Cluster>();
for (Cluster c : clustersList.get(iteration)) {
- next.add(new Cluster(c.getCenter(), c.getId()));
+ next.add(new Cluster(c.getCenter(), c.getId(), measure));
}
clustersList.add(next);
converged = runKMeansIteration(points, next, measure, distanceThreshold);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java Wed Aug 18 21:47:30 2010
@@ -63,8 +63,8 @@ public class KMeansDriver extends Abstra
* the directory pathname for initial & computed clusters
* @param output
* the directory pathname for output points
- * @param measureClass
- * the classname of the DistanceMeasure
+ * @param measure
+ * the DistanceMeasure
* @param convergenceDelta
* the convergence delta value
* @param maxIterations
@@ -78,17 +78,17 @@ public class KMeansDriver extends Abstra
public static void runJob(Path input,
Path clustersIn,
Path output,
- String measureClass,
+ DistanceMeasure measure,
double convergenceDelta,
int maxIterations,
int numReduceTasks,
boolean runClustering,
- boolean runSequential)
- throws IOException, InterruptedException, ClassNotFoundException, InstantiationException, IllegalAccessException {
+ boolean runSequential) throws IOException, InterruptedException, ClassNotFoundException,
+ InstantiationException, IllegalAccessException {
new KMeansDriver().job(input,
clustersIn,
output,
- measureClass,
+ measure,
convergenceDelta,
maxIterations,
numReduceTasks,
@@ -104,8 +104,7 @@ public class KMeansDriver extends Abstra
addOption(DefaultOptionCreator.distanceMeasureOption().create());
addOption(DefaultOptionCreator.clustersInOption()
.withDescription("The input centroids, as Vectors. Must be a SequenceFile of Writable, Cluster/Canopy. "
- + "If k is also specified, then a random set of vectors will be selected"
- + " and written out to this path first")
+ + "If k is also specified, then a random set of vectors will be selected" + " and written out to this path first")
.create());
addOption(DefaultOptionCreator.numClustersOption()
.withDescription("The k in k-Means. If specified, then a random selection of k Vectors will be chosen"
@@ -134,22 +133,16 @@ public class KMeansDriver extends Abstra
if (hasOption(DefaultOptionCreator.OVERWRITE_OPTION)) {
HadoopUtil.overwriteOutput(output);
}
+ ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+ DistanceMeasure measure = (DistanceMeasure) ((Class<?>) ccl.loadClass(measureClass)).newInstance();
+
if (hasOption(DefaultOptionCreator.NUM_CLUSTERS_OPTION)) {
clusters = RandomSeedGenerator.buildRandom(input, clusters, Integer
- .parseInt(getOption(DefaultOptionCreator.NUM_CLUSTERS_OPTION)));
+ .parseInt(getOption(DefaultOptionCreator.NUM_CLUSTERS_OPTION)), measure);
}
boolean runClustering = hasOption(DefaultOptionCreator.CLUSTERING_OPTION);
- boolean runSequential =
- getOption(DefaultOptionCreator.METHOD_OPTION).equalsIgnoreCase(DefaultOptionCreator.SEQUENTIAL_METHOD);
- job(input,
- clusters,
- output,
- measureClass,
- convergenceDelta,
- maxIterations,
- numReduceTasks,
- runClustering,
- runSequential);
+ boolean runSequential = getOption(DefaultOptionCreator.METHOD_OPTION).equalsIgnoreCase(DefaultOptionCreator.SEQUENTIAL_METHOD);
+ job(input, clusters, output, measure, convergenceDelta, maxIterations, numReduceTasks, runClustering, runSequential);
return 0;
}
@@ -163,8 +156,7 @@ public class KMeansDriver extends Abstra
* the directory pathname for initial & computed clusters
* @param output
* the directory pathname for output points
- * @param measureClass
- * the classname of the DistanceMeasure
+ * @param measure TODO
* @param convergenceDelta
* the convergence delta value
* @param maxIterations
@@ -178,39 +170,26 @@ public class KMeansDriver extends Abstra
public void job(Path input,
Path clustersIn,
Path output,
- String measureClass,
+ DistanceMeasure measure,
double convergenceDelta,
int maxIterations,
int numReduceTasks,
boolean runClustering,
- boolean runSequential)
- throws IOException, InterruptedException, ClassNotFoundException, InstantiationException,
+ boolean runSequential) throws IOException, InterruptedException, ClassNotFoundException, InstantiationException,
IllegalAccessException {
- ClassLoader ccl = Thread.currentThread().getContextClassLoader();
- Class<?> cl = ccl.loadClass(measureClass);
- DistanceMeasure measure = (DistanceMeasure) cl.newInstance();
// iterate until the clusters converge
String delta = Double.toString(convergenceDelta);
if (log.isInfoEnabled()) {
- log.info("Input: {} Clusters In: {} Out: {} Distance: {}",
- new Object[] { input, clustersIn, output, measureClass });
- log.info("convergence: {} max Iterations: {} num Reduce Tasks: {} Input Vectors: {}",
- new Object[] { convergenceDelta, maxIterations, numReduceTasks, VectorWritable.class.getName() });
- }
- Path clustersOut = buildClusters(input,
- clustersIn,
- output,
- measure,
- maxIterations,
- numReduceTasks,
- delta,
- runSequential);
+ log.info("Input: {} Clusters In: {} Out: {} Distance: {}", new Object[] { input, clustersIn, output,
+ measure.getClass().getName() });
+ log.info("convergence: {} max Iterations: {} num Reduce Tasks: {} Input Vectors: {}", new Object[] { convergenceDelta,
+ maxIterations, numReduceTasks, VectorWritable.class.getName() });
+ }
+ Path clustersOut = buildClusters(input, clustersIn, output, measure, maxIterations, numReduceTasks, delta, runSequential);
if (runClustering) {
log.info("Clustering data");
- clusterData(input,
- clustersOut,
- new Path(output, AbstractCluster.CLUSTERED_POINTS_DIR), measure, delta, runSequential);
+ clusterData(input, clustersOut, new Path(output, AbstractCluster.CLUSTERED_POINTS_DIR), measure, delta, runSequential);
}
}
@@ -251,13 +230,12 @@ public class KMeansDriver extends Abstra
}
private Path buildClustersSeq(Path input,
- Path clustersIn,
- Path output,
- DistanceMeasure measure,
- int maxIterations,
- int numReduceTasks,
- String delta)
- throws InstantiationException, IllegalAccessException, IOException {
+ Path clustersIn,
+ Path output,
+ DistanceMeasure measure,
+ int maxIterations,
+ int numReduceTasks,
+ String delta) throws InstantiationException, IllegalAccessException, IOException {
KMeansClusterer clusterer = new KMeansClusterer(measure);
List<Cluster> clusters = new ArrayList<Cluster>();
@@ -294,12 +272,9 @@ public class KMeansDriver extends Abstra
Cluster.class);
try {
for (Cluster cluster : clusters) {
- log.info("Writing Cluster:{} center:{} numPoints:{} radius:{} to: {}",
- new Object[] { cluster.getId(),
- AbstractCluster.formatVector(cluster.getCenter(), null),
- cluster.getNumPoints(),
- AbstractCluster.formatVector(cluster.getRadius(), null),
- clustersOut.getName() });
+ log.info("Writing Cluster:{} center:{} numPoints:{} radius:{} to: {}", new Object[] { cluster.getId(),
+ AbstractCluster.formatVector(cluster.getCenter(), null), cluster.getNumPoints(),
+ AbstractCluster.formatVector(cluster.getRadius(), null), clustersOut.getName() });
writer.append(new Text(cluster.getIdentifier()), cluster);
}
} finally {
@@ -350,12 +325,11 @@ public class KMeansDriver extends Abstra
* @return true if the iteration successfully runs
*/
private boolean runIteration(Path input,
- Path clustersIn,
- Path clustersOut,
- String measureClass,
- String convergenceDelta,
- int numReduceTasks)
- throws IOException, InterruptedException, ClassNotFoundException {
+ Path clustersIn,
+ Path clustersOut,
+ String measureClass,
+ String convergenceDelta,
+ int numReduceTasks) throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set(KMeansConfigKeys.CLUSTER_PATH_KEY, clustersIn.toString());
conf.set(KMeansConfigKeys.DISTANCE_MEASURE_KEY, measureClass);
@@ -461,11 +435,7 @@ public class KMeansDriver extends Abstra
}
}
- private void clusterDataSeq(Path input,
- Path clustersIn,
- Path output,
- DistanceMeasure measure,
- String convergenceDelta)
+ private void clusterDataSeq(Path input, Path clustersIn, Path output, DistanceMeasure measure, String convergenceDelta)
throws IOException, InterruptedException, InstantiationException, IllegalAccessException {
KMeansClusterer clusterer = new KMeansClusterer(measure);
List<Cluster> clusters = new ArrayList<Cluster>();
@@ -499,11 +469,7 @@ public class KMeansDriver extends Abstra
}
- private void clusterDataMR(Path input,
- Path clustersIn,
- Path output,
- DistanceMeasure measure,
- String convergenceDelta)
+ private void clusterDataMR(Path input, Path clustersIn, Path output, DistanceMeasure measure, String convergenceDelta)
throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set(KMeansConfigKeys.CLUSTER_PATH_KEY, clustersIn.toString());
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansUtil.java Wed Aug 18 21:47:30 2010
@@ -35,19 +35,20 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
final class KMeansUtil {
-
+
private static final Logger log = LoggerFactory.getLogger(KMeansUtil.class);
-
- private KMeansUtil() { }
-
+
+ private KMeansUtil() {
+ }
+
/** Configure the mapper with the cluster info */
public static void configureWithClusterInfo(Path clusterPathStr, List<Cluster> clusters) {
-
+
// Get the path location where the cluster Info is stored
Configuration conf = new Configuration();
Path clusterPath = new Path(clusterPathStr, "*");
List<Path> result = new ArrayList<Path>();
-
+
// filter out the files
PathFilter clusterFileFilter = new PathFilter() {
@Override
@@ -55,17 +56,16 @@ final class KMeansUtil {
return path.getName().startsWith("part");
}
};
-
+
try {
// get all filtered file names in result list
FileSystem fs = clusterPath.getFileSystem(conf);
- FileStatus[] matches = fs.listStatus(
- FileUtil.stat2Paths(fs.globStatus(clusterPath, clusterFileFilter)), clusterFileFilter);
-
+ FileStatus[] matches = fs.listStatus(FileUtil.stat2Paths(fs.globStatus(clusterPath, clusterFileFilter)), clusterFileFilter);
+
for (FileStatus match : matches) {
result.add(fs.makeQualified(match.getPath()));
}
-
+
// iterate thru the result path list
for (Path path : result) {
SequenceFile.Reader reader = null;
@@ -93,7 +93,7 @@ final class KMeansUtil {
Canopy value = new Canopy();
while (reader.next(key, value)) {
// get the cluster info
- Cluster cluster = new Cluster(value.getCenter(), value.getId());
+ Cluster cluster = new Cluster(value.getCenter(), value.getId(), value.getMeasure());
clusters.add(cluster);
value = new Canopy();
}
@@ -102,11 +102,11 @@ final class KMeansUtil {
IOUtils.quietClose(reader);
}
}
-
+
} catch (IOException e) {
log.info("Exception occurred in loading clusters:", e);
throw new IllegalStateException(e);
}
}
-
+
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java Wed Aug 18 21:47:30 2010
@@ -31,6 +31,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.VectorWritable;
import org.slf4j.Logger;
@@ -51,7 +52,7 @@ public final class RandomSeedGenerator {
private RandomSeedGenerator() {
}
- public static Path buildRandom(Path input, Path output, int k) throws IOException,
+ public static Path buildRandom(Path input, Path output, int k, DistanceMeasure measure) throws IOException,
IllegalAccessException,
InstantiationException {
// delete the output directory
@@ -84,7 +85,7 @@ public final class RandomSeedGenerator {
Writable key = (Writable) reader.getKeyClass().newInstance();
VectorWritable value = (VectorWritable) reader.getValueClass().newInstance();
while (reader.next(key, value)) {
- Cluster newCluster = new Cluster(value.get(), nextClusterId++);
+ Cluster newCluster = new Cluster(value.get(), nextClusterId++, measure);
newCluster.observe(value.get(), 1);
Text newText = new Text(key.toString());
int currentSize = chosenTexts.size();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java Wed Aug 18 21:47:30 2010
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.lang.reflect.Type;
import org.apache.mahout.clustering.kmeans.Cluster;
+import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.math.JsonVectorAdapter;
import org.apache.mahout.math.Vector;
import org.apache.mahout.math.list.IntArrayList;
@@ -53,11 +54,12 @@ public class MeanShiftCanopy extends Clu
/**
* Create a new Canopy containing the given point
*
- * @param point
- * a Vector
+ * @param point a Vector
+ * @param id an int canopy id
+ * @param measure a DistanceMeasure
*/
- public MeanShiftCanopy(Vector point, int id) {
- super(point, id);
+ public MeanShiftCanopy(Vector point, int id, DistanceMeasure measure) {
+ super(point, id, measure);
boundPoints.add(id);
}
@@ -128,6 +130,7 @@ public class MeanShiftCanopy extends Clu
public MeanShiftCanopy shallowCopy() {
MeanShiftCanopy result = new MeanShiftCanopy();
+ result.setMeasure(this.getMeasure());
result.setId(this.getId());
result.setCenter(this.getCenter());
result.setRadius(this.getRadius());
@@ -138,7 +141,10 @@ public class MeanShiftCanopy extends Clu
@Override
public String asFormatString() {
- return formatCanopy(this);
+ GsonBuilder gBuilder = new GsonBuilder();
+ gBuilder.registerTypeAdapter(VECTOR_TYPE, new JsonVectorAdapter());
+ Gson gson = gBuilder.create();
+ return gson.toJson(this, MeanShiftCanopy.class);
}
public void setBoundPoints(IntArrayList boundPoints) {
@@ -150,26 +156,4 @@ public class MeanShiftCanopy extends Clu
return (isConverged() ? "MSV-" : "MSC-") + getId();
}
- /** Format the canopy for output */
- public static String formatCanopy(MeanShiftCanopy canopy) {
- GsonBuilder gBuilder = new GsonBuilder();
- gBuilder.registerTypeAdapter(VECTOR_TYPE, new JsonVectorAdapter());
- Gson gson = gBuilder.create();
- return gson.toJson(canopy, MeanShiftCanopy.class);
- }
-
- /**
- * Decodes and returns a Canopy from the formattedString
- *
- * @param formattedString
- * a String produced by formatCanopy
- * @return a new Canopy
- */
- public static MeanShiftCanopy decodeCanopy(String formattedString) {
- GsonBuilder gBuilder = new GsonBuilder();
- gBuilder.registerTypeAdapter(VECTOR_TYPE, new JsonVectorAdapter());
- Gson gson = gBuilder.create();
- return gson.fromJson(formattedString, MeanShiftCanopy.class);
- }
-
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java Wed Aug 18 21:47:30 2010
@@ -169,7 +169,7 @@ public class MeanShiftCanopyClusterer {
List<MeanShiftCanopy> canopies = new ArrayList<MeanShiftCanopy>();
for (Vector point : points) {
- clusterer.mergeCanopy(new MeanShiftCanopy(point, nextCanopyId++), canopies);
+ clusterer.mergeCanopy(new MeanShiftCanopy(point, nextCanopyId++, measure), canopies);
}
List<MeanShiftCanopy> newCanopies = canopies;
boolean[] converged = { false };
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java?rev=986960&r1=986959&r2=986960&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java Wed Aug 18 21:47:30 2010
@@ -23,6 +23,8 @@ import java.util.regex.Pattern;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.mahout.clustering.kmeans.KMeansConfigKeys;
+import org.apache.mahout.common.distance.DistanceMeasure;
import org.apache.mahout.math.VectorWritable;
public class MeanShiftCanopyCreatorMapper extends Mapper<WritableComparable<?>, VectorWritable, Text, MeanShiftCanopy> {
@@ -31,15 +33,29 @@ public class MeanShiftCanopyCreatorMappe
private static int nextCanopyId = -1;
+ private DistanceMeasure measure;
+
@Override
protected void map(WritableComparable<?> key, VectorWritable point, Context context) throws IOException, InterruptedException {
- MeanShiftCanopy canopy = new MeanShiftCanopy(point.get(), nextCanopyId++);
+ MeanShiftCanopy canopy = new MeanShiftCanopy(point.get(), nextCanopyId++, measure);
context.write(new Text(key.toString()), canopy);
}
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
+ String measureClass = context.getConfiguration().get(KMeansConfigKeys.DISTANCE_MEASURE_KEY);
+ ClassLoader ccl = Thread.currentThread().getContextClassLoader();
+ try {
+ measure = (DistanceMeasure) ((Class<?>) ccl.loadClass(measureClass)).newInstance();
+ } catch (InstantiationException e) {
+ new IllegalStateException(e);
+ } catch (IllegalAccessException e) {
+ new IllegalStateException(e);
+ } catch (ClassNotFoundException e) {
+ new IllegalStateException(e);
+ }
+
if (nextCanopyId == -1) {
String taskId = context.getConfiguration().get("mapred.task.id");
String[] parts = UNDERSCORE_PATTERN.split(taskId);