You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2011/04/17 17:30:17 UTC
svn commit: r1094158 [2/3] - in /mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/hadoop/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/als/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/
core/src/main/java/org/apache/maho...
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletClusterMapper.java Sun Apr 17 15:30:15 2011
@@ -65,9 +65,11 @@ public class DirichletClusterMapper
protected static List<DirichletCluster> loadClusters(Configuration conf, Path stateIn) {
List<DirichletCluster> clusters = new ArrayList<DirichletCluster>();
- for (DirichletCluster value :
- new SequenceFileDirValueIterable<DirichletCluster>(
- stateIn, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (DirichletCluster value
+ : new SequenceFileDirValueIterable<DirichletCluster>(stateIn,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
clusters.add(value);
}
return clusters;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletDriver.java Sun Apr 17 15:30:15 2011
@@ -320,7 +320,7 @@ public class DirichletDriver extends Abs
FileOutputFormat.setOutputPath(job, stateOut);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Dirichlet Iteration failed processing " + stateIn.toString());
+ throw new InterruptedException("Dirichlet Iteration failed processing " + stateIn);
}
}
@@ -389,9 +389,11 @@ public class DirichletDriver extends Abs
newModel.configure(conf);
}
DirichletClusterer clusterer = new DirichletClusterer(state);
- for (VectorWritable value :
- new SequenceFileDirValueIterable<VectorWritable>(
- input, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (VectorWritable value
+ : new SequenceFileDirValueIterable<VectorWritable>(input,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
clusterer.observe(newModels, value);
}
clusterer.updateModels(newModels);
@@ -511,7 +513,7 @@ public class DirichletDriver extends Abs
FileOutputFormat.setOutputPath(job, output);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Dirichlet Clustering failed processing " + stateIn.toString());
+ throw new InterruptedException("Dirichlet Clustering failed processing " + stateIn);
}
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/dirichlet/DirichletMapper.java Sun Apr 17 15:30:15 2011
@@ -80,9 +80,11 @@ public class DirichletMapper extends Map
int k) {
DirichletState state = DirichletDriver.createState(description, k, alpha);
Path path = new Path(statePath);
- for (Pair<Writable,DirichletCluster> record :
- new SequenceFileDirIterable<Writable,DirichletCluster>(
- path, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (Pair<Writable,DirichletCluster> record
+ : new SequenceFileDirIterable<Writable,DirichletCluster>(path,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
int index = Integer.parseInt(record.getFirst().toString());
state.getClusters().set(index, record.getSecond());
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansCombiner.java Sun Apr 17 15:30:15 2011
@@ -28,7 +28,8 @@ public class FuzzyKMeansCombiner extends
private FuzzyKMeansClusterer clusterer;
@Override
- protected void reduce(Text key, Iterable<ClusterObservations> values, Context context) throws IOException, InterruptedException {
+ protected void reduce(Text key, Iterable<ClusterObservations> values, Context context)
+ throws IOException, InterruptedException {
SoftCluster cluster = new SoftCluster();
for (ClusterObservations value : values) {
if (value.getCombinerState() == 0) { // first time thru combiner
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java Sun Apr 17 15:30:15 2011
@@ -245,7 +245,7 @@ public class FuzzyKMeansDriver extends A
FileOutputFormat.setOutputPath(job, clustersOut);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Fuzzy K-Means Iteration failed processing " + clustersIn.toString());
+ throw new InterruptedException("Fuzzy K-Means Iteration failed processing " + clustersIn);
}
FileSystem fs = FileSystem.get(clustersOut.toUri(), conf);
return isConverged(clustersOut, conf, fs);
@@ -362,9 +362,11 @@ public class FuzzyKMeansDriver extends A
log.info("Fuzzy k-Means Iteration: " + iteration);
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(input.toUri(), conf);
- for (VectorWritable value :
- new SequenceFileDirValueIterable<VectorWritable>(
- input, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (VectorWritable value
+ : new SequenceFileDirValueIterable<VectorWritable>(input,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
clusterer.addPointToClusters(clusters,value.get());
}
converged = clusterer.testConvergence(clusters);
@@ -377,7 +379,8 @@ public class FuzzyKMeansDriver extends A
try {
for (SoftCluster cluster : clusters) {
log.debug("Writing Cluster:{} center:{} numPoints:{} radius:{} to: {}",
- new Object[] { cluster.getId(),
+ new Object[] {
+ cluster.getId(),
AbstractCluster.formatVector(cluster.getCenter(), null),
cluster.getNumPoints(),
AbstractCluster.formatVector(cluster.getRadius(), null),
@@ -523,7 +526,7 @@ public class FuzzyKMeansDriver extends A
job.setJarByClass(FuzzyKMeansDriver.class);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Fuzzy K-Means Clustering failed processing " + clustersIn.toString());
+ throw new InterruptedException("Fuzzy K-Means Clustering failed processing " + clustersIn);
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansMapper.java Sun Apr 17 15:30:15 2011
@@ -40,7 +40,8 @@ public class FuzzyKMeansMapper extends M
private FuzzyKMeansClusterer clusterer;
@Override
- protected void map(WritableComparable<?> key, VectorWritable point, Context context) throws IOException, InterruptedException {
+ protected void map(WritableComparable<?> key, VectorWritable point, Context context)
+ throws IOException, InterruptedException {
clusterer.emitPointProbToCluster(point.get(), clusters, context);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansReducer.java Sun Apr 17 15:30:15 2011
@@ -36,7 +36,8 @@ public class FuzzyKMeansReducer extends
private FuzzyKMeansClusterer clusterer;
@Override
- protected void reduce(Text key, Iterable<ClusterObservations> values, Context context) throws IOException, InterruptedException {
+ protected void reduce(Text key, Iterable<ClusterObservations> values, Context context)
+ throws IOException, InterruptedException {
SoftCluster cluster = clusterMap.get(key.toString());
for (ClusterObservations value : values) {
if (value.getCombinerState() == 0) { // escaped from combiner
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansClusterMapper.java Sun Apr 17 15:30:15 2011
@@ -31,13 +31,14 @@ import org.apache.mahout.common.distance
import org.apache.mahout.math.VectorWritable;
/**
- * The {@link org.apache.mahout.clustering.kmeans.KMeansClusterMapper} is responsible for calculating which points belong
- * to which clusters and outputting the information. This is an optional step, as some applications only care about
- * what the Centroids are.
+ * The {@link org.apache.mahout.clustering.kmeans.KMeansClusterMapper} is responsible for calculating
+ * which points belong to which clusters and outputting the information. This is an optional step,
+ * as some applications only care about what the Centroids are.
*
* @see KMeansDriver for more information on how to invoke this process
*/
-public class KMeansClusterMapper extends Mapper<WritableComparable<?>,VectorWritable,IntWritable,WeightedVectorWritable> {
+public class KMeansClusterMapper
+ extends Mapper<WritableComparable<?>,VectorWritable,IntWritable,WeightedVectorWritable> {
private final Collection<Cluster> clusters = new ArrayList<Cluster>();
private KMeansClusterer clusterer;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansCombiner.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansCombiner.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansCombiner.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansCombiner.java Sun Apr 17 15:30:15 2011
@@ -25,8 +25,8 @@ import org.apache.mahout.clustering.Clus
public class KMeansCombiner extends Reducer<Text, ClusterObservations, Text, ClusterObservations> {
@Override
- protected void reduce(Text key, Iterable<ClusterObservations> values, Context context) throws IOException, InterruptedException {
-
+ protected void reduce(Text key, Iterable<ClusterObservations> values, Context context)
+ throws IOException, InterruptedException {
Cluster cluster = new Cluster();
for (ClusterObservations value : values) {
cluster.observe(value);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansDriver.java Sun Apr 17 15:30:15 2011
@@ -104,7 +104,7 @@ public class KMeansDriver extends Abstra
boolean runClustering = hasOption(DefaultOptionCreator.CLUSTERING_OPTION);
boolean runSequential = getOption(DefaultOptionCreator.METHOD_OPTION).equalsIgnoreCase(
DefaultOptionCreator.SEQUENTIAL_METHOD);
- if(getConf() == null) {
+ if (getConf() == null) {
setConf(new Configuration());
}
run(getConf(), input, clusters, output, measure, convergenceDelta, maxIterations, runClustering, runSequential);
@@ -144,10 +144,10 @@ public class KMeansDriver extends Abstra
// iterate until the clusters converge
String delta = Double.toString(convergenceDelta);
if (log.isInfoEnabled()) {
- log.info("Input: {} Clusters In: {} Out: {} Distance: {}", new Object[] { input, clustersIn, output,
- measure.getClass().getName() });
+ log.info("Input: {} Clusters In: {} Out: {} Distance: {}",
+ new Object[] {input, clustersIn, output,measure.getClass().getName()});
log.info("convergence: {} max Iterations: {} num Reduce Tasks: {} Input Vectors: {}",
- new Object[] { convergenceDelta, maxIterations, VectorWritable.class.getName() });
+ new Object[] {convergenceDelta, maxIterations, VectorWritable.class.getName()});
}
Path clustersOut = buildClusters(conf, input, clustersIn, output, measure, maxIterations, delta, runSequential);
if (runClustering) {
@@ -259,9 +259,11 @@ public class KMeansDriver extends Abstra
while (!converged && iteration <= maxIterations) {
log.info("K-Means Iteration: " + iteration);
FileSystem fs = FileSystem.get(input.toUri(), conf);
- for (VectorWritable value :
- new SequenceFileDirValueIterable<VectorWritable>(
- input, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (VectorWritable value
+ : new SequenceFileDirValueIterable<VectorWritable>(input,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
clusterer.addPointToNearestCluster(value.get(), clusters);
}
converged = clusterer.testConvergence(clusters, Double.parseDouble(delta));
@@ -273,9 +275,13 @@ public class KMeansDriver extends Abstra
Cluster.class);
try {
for (Cluster cluster : clusters) {
- log.debug("Writing Cluster:{} center:{} numPoints:{} radius:{} to: {}", new Object[] { cluster.getId(),
- AbstractCluster.formatVector(cluster.getCenter(), null), cluster.getNumPoints(),
- AbstractCluster.formatVector(cluster.getRadius(), null), clustersOut.getName() });
+ log.debug("Writing Cluster:{} center:{} numPoints:{} radius:{} to: {}",
+ new Object[] {
+ cluster.getId(),
+ AbstractCluster.formatVector(cluster.getCenter(), null),
+ cluster.getNumPoints(),
+ AbstractCluster.formatVector(cluster.getRadius(), null), clustersOut.getName()
+ });
writer.append(new Text(cluster.getIdentifier()), cluster);
}
} finally {
@@ -354,7 +360,7 @@ public class KMeansDriver extends Abstra
job.setJarByClass(KMeansDriver.class);
HadoopUtil.delete(conf, clustersOut);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("K-Means Iteration failed processing " + clustersIn.toString());
+ throw new InterruptedException("K-Means Iteration failed processing " + clustersIn);
}
FileSystem fs = FileSystem.get(clustersOut.toUri(), conf);
@@ -409,7 +415,7 @@ public class KMeansDriver extends Abstra
if (log.isInfoEnabled()) {
log.info("Running Clustering");
- log.info("Input: {} Clusters In: {} Out: {} Distance: {}", new Object[] { input, clustersIn, output, measure });
+ log.info("Input: {} Clusters In: {} Out: {} Distance: {}", new Object[] {input, clustersIn, output, measure});
log.info("convergence: {} Input Vectors: {}", convergenceDelta, VectorWritable.class.getName());
}
if (runSequential) {
@@ -478,7 +484,7 @@ public class KMeansDriver extends Abstra
job.setJarByClass(KMeansDriver.class);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("K-Means Clustering failed processing " + clustersIn.toString());
+ throw new InterruptedException("K-Means Clustering failed processing " + clustersIn);
}
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/RandomSeedGenerator.java Sun Apr 17 15:30:15 2011
@@ -83,8 +83,8 @@ public final class RandomSeedGenerator {
if (fileStatus.isDir()) {
continue;
}
- for (Pair<Writable,VectorWritable> record :
- new SequenceFileIterable<Writable,VectorWritable>(fileStatus.getPath(), true, conf)) {
+ for (Pair<Writable,VectorWritable> record
+ : new SequenceFileIterable<Writable,VectorWritable>(fileStatus.getPath(), true, conf)) {
Writable key = record.getFirst();
VectorWritable value = record.getSecond();
Cluster newCluster = new Cluster(value.get(), nextClusterId++, measure);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDADriver.java Sun Apr 17 15:30:15 2011
@@ -86,9 +86,13 @@ public final class LDADriver extends Abs
double[] logTotals = new double[numTopics];
double ll = 0.0;
- for (Pair<IntPairWritable,DoubleWritable> record :
- new SequenceFileDirIterable<IntPairWritable, DoubleWritable>(
- new Path(dir, "part-*"), PathType.GLOB, null, null, true, job)) {
+ for (Pair<IntPairWritable,DoubleWritable> record
+ : new SequenceFileDirIterable<IntPairWritable, DoubleWritable>(new Path(dir, "part-*"),
+ PathType.GLOB,
+ null,
+ null,
+ true,
+ job)) {
IntPairWritable key = record.getFirst();
DoubleWritable value = record.getSecond();
int topic = key.getFirst();
@@ -263,7 +267,7 @@ public final class LDADriver extends Abs
job.setJarByClass(LDADriver.class);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("LDA Iteration failed processing " + stateIn.toString());
+ throw new InterruptedException("LDA Iteration failed processing " + stateIn);
}
return findLL(stateOut, conf);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAReducer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/lda/LDAReducer.java Sun Apr 17 15:30:15 2011
@@ -38,7 +38,8 @@ public class LDAReducer extends Reducer<
double accum = 0.0;
for (DoubleWritable vw : values) {
double v = vw.get();
- Preconditions.checkArgument(!Double.isNaN(v), "Found NaN for topic=(%d,%d)", topicWord.getFirst(), topicWord.getSecond());
+ Preconditions.checkArgument(!Double.isNaN(v),
+ "Found NaN for topic=(%d,%d)", topicWord.getFirst(), topicWord.getSecond());
accum += v;
}
context.write(topicWord, new DoubleWritable(accum));
@@ -46,9 +47,11 @@ public class LDAReducer extends Reducer<
double accum = Double.NEGATIVE_INFINITY;
for (DoubleWritable vw : values) {
double v = vw.get();
- Preconditions.checkArgument(!Double.isNaN(v), "Found NaN for topic = (%d,%d)", topicWord.getFirst(), topicWord.getSecond());
+ Preconditions.checkArgument(!Double.isNaN(v),
+ "Found NaN for topic = (%d,%d)", topicWord.getFirst(), topicWord.getSecond());
accum = LDAUtil.logSum(accum, v);
- Preconditions.checkArgument(!Double.isNaN(accum), "Accumulated NaN for topic = (%d,%d)", topicWord.getFirst(), topicWord.getSecond());
+ Preconditions.checkArgument(!Double.isNaN(accum),
+ "Accumulated NaN for topic = (%d,%d)", topicWord.getFirst(), topicWord.getSecond());
}
context.write(topicWord, new DoubleWritable(accum));
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java Sun Apr 17 15:30:15 2011
@@ -63,11 +63,11 @@ public class MeanShiftCanopy extends Clu
* @param measure a DistanceMeasure
* @return a MeanShiftCanopy
*/
- public static MeanShiftCanopy initialCanopy(Vector point, int id, DistanceMeasure measure){
- MeanShiftCanopy result = new MeanShiftCanopy(point, id, measure);
- // overwrite center so original point type is retained
- result.setCenter(point);
- return result;
+ public static MeanShiftCanopy initialCanopy(Vector point, int id, DistanceMeasure measure) {
+ MeanShiftCanopy result = new MeanShiftCanopy(point, id, measure);
+ // overwrite center so original point type is retained
+ result.setCenter(point);
+ return result;
}
/**
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterMapper.java Sun Apr 17 15:30:15 2011
@@ -61,8 +61,8 @@ public class MeanShiftCanopyClusterMappe
String statePath = conf.get(MeanShiftCanopyDriver.STATE_IN_KEY);
List<MeanShiftCanopy> canopies = new ArrayList<MeanShiftCanopy>();
Path path = new Path(statePath);
- for (MeanShiftCanopy value :
- new SequenceFileDirValueIterable<MeanShiftCanopy>(path, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (MeanShiftCanopy value
+ : new SequenceFileDirValueIterable<MeanShiftCanopy>(path, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
canopies.add(value);
}
return canopies;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyClusterer.java Sun Apr 17 15:30:15 2011
@@ -172,7 +172,7 @@ public class MeanShiftCanopyClusterer {
clusterer.mergeCanopy(new MeanShiftCanopy(point, nextCanopyId++, measure), canopies);
}
List<MeanShiftCanopy> newCanopies = canopies;
- boolean[] converged = { false };
+ boolean[] converged = {false};
for (int iter = 0; !converged[0] && iter < numIter; iter++) {
newCanopies = clusterer.iterate(newCanopies, converged);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCreatorMapper.java Sun Apr 17 15:30:15 2011
@@ -37,7 +37,8 @@ public class MeanShiftCanopyCreatorMappe
private DistanceMeasure measure;
@Override
- protected void map(WritableComparable<?> key, VectorWritable point, Context context) throws IOException, InterruptedException {
+ protected void map(WritableComparable<?> key, VectorWritable point, Context context)
+ throws IOException, InterruptedException {
MeanShiftCanopy canopy = MeanShiftCanopy.initialCanopy(point.get(), nextCanopyId++, measure);
context.write(new Text(key.toString()), canopy);
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyDriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyDriver.java Sun Apr 17 15:30:15 2011
@@ -236,7 +236,7 @@ public class MeanShiftCanopyDriver exten
FileOutputFormat.setOutputPath(job, output);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Mean Shift createCanopyFromVectorsMR failed on input " + input.toString());
+ throw new InterruptedException("Mean Shift createCanopyFromVectorsMR failed on input " + input);
}
}
@@ -291,9 +291,11 @@ public class MeanShiftCanopyDriver exten
List<MeanShiftCanopy> clusters = new ArrayList<MeanShiftCanopy>();
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(clustersIn.toUri(), conf);
- for (MeanShiftCanopy value :
- new SequenceFileDirValueIterable<MeanShiftCanopy>(
- clustersIn, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (MeanShiftCanopy value
+ : new SequenceFileDirValueIterable<MeanShiftCanopy>(clustersIn,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
clusterer.mergeCanopy(value, clusters);
}
boolean[] converged = {false};
@@ -407,7 +409,7 @@ public class MeanShiftCanopyDriver exten
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setJarByClass(MeanShiftCanopyDriver.class);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Mean Shift Iteration failed on input " + input.toString());
+ throw new InterruptedException("Mean Shift Iteration failed on input " + input);
}
}
@@ -436,8 +438,11 @@ public class MeanShiftCanopyDriver exten
private static void clusterDataSeq(Path input, Path clustersIn, Path output) throws IOException {
Collection<MeanShiftCanopy> clusters = new ArrayList<MeanShiftCanopy>();
Configuration conf = new Configuration();
- for (MeanShiftCanopy value :
- new SequenceFileDirValueIterable<MeanShiftCanopy>(clustersIn, PathType.LIST, PathFilters.logsCRCFilter(), conf)) {
+ for (MeanShiftCanopy value
+ : new SequenceFileDirValueIterable<MeanShiftCanopy>(clustersIn,
+ PathType.LIST,
+ PathFilters.logsCRCFilter(),
+ conf)) {
clusters.add(value);
}
// iterate over all points, assigning each to the closest canopy and outputting that clustering
@@ -451,8 +456,8 @@ public class MeanShiftCanopyDriver exten
IntWritable.class,
WeightedVectorWritable.class);
try {
- for (Pair<Writable,MeanShiftCanopy> record :
- new SequenceFileIterable<Writable,MeanShiftCanopy>(s.getPath(), conf)) {
+ for (Pair<Writable,MeanShiftCanopy> record
+ : new SequenceFileIterable<Writable,MeanShiftCanopy>(s.getPath(), conf)) {
MeanShiftCanopy canopy = record.getSecond();
MeanShiftCanopy closest = MeanShiftCanopyClusterer.findCoveringCanopy(canopy, clusters);
writer.append(new IntWritable(closest.getId()), new WeightedVectorWritable(1, canopy.getCenter()));
@@ -484,7 +489,7 @@ public class MeanShiftCanopyDriver exten
FileOutputFormat.setOutputPath(job, output);
if (!job.waitForCompletion(true)) {
- throw new InterruptedException("Mean Shift Clustering failed on clustersIn " + clustersIn.toString());
+ throw new InterruptedException("Mean Shift Clustering failed on clustersIn " + clustersIn);
}
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/minhash/MinHashMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/minhash/MinHashMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/minhash/MinHashMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/minhash/MinHashMapper.java Sun Apr 17 15:30:15 2011
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
-public class MinHashMapper extends Mapper<Text,Writable,Text,Writable> {
+public class MinHashMapper extends Mapper<Text,VectorWritable,Text,Writable> {
private static final Logger log = LoggerFactory.getLogger(MinHashMapper.class);
@@ -72,8 +72,8 @@ public class MinHashMapper extends Mappe
* cluster-id as 'key' and item-id as 'value'
*/
@Override
- public void map(Text item, Writable features, Context context) throws IOException, InterruptedException {
- Vector featureVector = ((VectorWritable) features).get();
+ public void map(Text item, VectorWritable features, Context context) throws IOException, InterruptedException {
+ Vector featureVector = features.get();
if (featureVector.size() < minVectorSize) {
return;
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsAffinityCutsJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsAffinityCutsJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsAffinityCutsJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsAffinityCutsJob.java Sun Apr 17 15:30:15 2011
@@ -149,39 +149,39 @@ public final class EigencutsAffinityCuts
}
// do we zero out the values?
- VertexWritable out_i = new VertexWritable();
- VertexWritable out_j = new VertexWritable();
+ VertexWritable outI = new VertexWritable();
+ VertexWritable outJ = new VertexWritable();
if (zero) {
// increment the cut counter
context.getCounter(CUTSCOUNTER.NUM_CUTS).increment(1);
// we want the values to exist on the diagonal
- out_i.setCol(i);
- out_j.setCol(j);
+ outI.setCol(i);
+ outJ.setCol(j);
// also, set the old values to zero
- VertexWritable zero_i = new VertexWritable();
- VertexWritable zero_j = new VertexWritable();
- zero_i.setCol(j);
- zero_i.setValue(0);
- zero_j.setCol(i);
- zero_j.setValue(0);
- zero_i.setType("unimportant");
- zero_j.setType("unimportant");
- context.write(new Text(String.valueOf(i)), zero_i);
- context.write(new Text(String.valueOf(j)), zero_j);
+ VertexWritable zeroI = new VertexWritable();
+ VertexWritable zeroJ = new VertexWritable();
+ zeroI.setCol(j);
+ zeroI.setValue(0);
+ zeroJ.setCol(i);
+ zeroJ.setValue(0);
+ zeroI.setType("unimportant");
+ zeroJ.setType("unimportant");
+ context.write(new Text(String.valueOf(i)), zeroI);
+ context.write(new Text(String.valueOf(j)), zeroJ);
} else {
- out_i.setCol(j);
- out_j.setCol(i);
+ outI.setCol(j);
+ outJ.setCol(i);
}
// set the values and write them
- out_i.setValue(k);
- out_j.setValue(k);
- out_i.setType("unimportant");
- out_j.setType("unimportant");
- context.write(new Text(String.valueOf(i)), out_i);
- context.write(new Text(String.valueOf(j)), out_j);
+ outI.setValue(k);
+ outJ.setValue(k);
+ outI.setType("unimportant");
+ outJ.setType("unimportant");
+ context.write(new Text(String.valueOf(i)), outI);
+ context.write(new Text(String.valueOf(j)), outJ);
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsDriver.java Sun Apr 17 15:30:15 2011
@@ -132,7 +132,8 @@ public class EigencutsDriver extends Abs
int overshoot = (int) ((double) eigenrank * OVERSHOOT_MULTIPLIER);
List<Double> eigenValues = new ArrayList<Double>(overshoot);
Matrix eigenVectors = new DenseMatrix(overshoot, eigenrank);
- DistributedRowMatrix U = performEigenDecomposition(conf, L, eigenrank, overshoot, eigenValues, eigenVectors, outputCalc);
+ DistributedRowMatrix U =
+ performEigenDecomposition(conf, L, eigenrank, overshoot, eigenValues, eigenVectors, outputCalc);
U.setConf(new Configuration(conf));
eigenValues = eigenValues.subList(0, eigenrank);
@@ -154,7 +155,8 @@ public class EigencutsDriver extends Abs
// how many cuts were made?
if (numCuts > 0) {
// recalculate A
- A = new DistributedRowMatrix(input, new Path(outputTmp, Long.toString(System.nanoTime())), dimensions, dimensions);
+ A = new DistributedRowMatrix(input,
+ new Path(outputTmp, Long.toString(System.nanoTime())), dimensions, dimensions);
A.setConf(new Configuration());
}
} while (numCuts > 0);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityMapper.java Sun Apr 17 15:30:15 2011
@@ -124,7 +124,7 @@ public class EigencutsSensitivityMapper
* Functions.LOGARITHM.apply(Functions.POW.apply(eigenvalue, beta0) / 2));
double secondhalf =
- -Functions.POW.apply(((evi / Functions.SQRT.apply(diagi)) - (evj / Functions.SQRT.apply(diagj))), 2)
+ -Functions.POW.apply((evi / Functions.SQRT.apply(diagi)) - (evj / Functions.SQRT.apply(diagj)), 2)
+ (1.0 - eigenvalue) * ((Functions.POW.apply(evi, 2) / diagi) + (Functions.POW.apply(evj, 2) / diagj));
return firsthalf * secondhalf;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityReducer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/eigencuts/EigencutsSensitivityReducer.java Sun Apr 17 15:30:15 2011
@@ -40,8 +40,8 @@ public class EigencutsSensitivityReducer
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
Vector v = new RandomAccessSparseVector(conf.getInt(EigencutsKeys.AFFINITY_DIMENSIONS, Integer.MAX_VALUE), 100);
- double threshold = Double.parseDouble(conf.get(EigencutsKeys.TAU)) /
- Double.parseDouble(conf.get(EigencutsKeys.DELTA));
+ double threshold = Double.parseDouble(conf.get(EigencutsKeys.TAU))
+ / Double.parseDouble(conf.get(EigencutsKeys.DELTA));
for (EigencutsSensitivityNode n : arr) {
if (n.getSensitivity() < threshold && n.getSensitivity() < v.getQuick(n.getColumn())) {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/clustering/spectral/kmeans/SpectralKMeansDriver.java Sun Apr 17 15:30:15 2011
@@ -190,14 +190,22 @@ public class SpectralKMeansDriver extend
new Path(output, Cluster.INITIAL_CLUSTERS_DIR),
clusters,
measure);
- KMeansDriver.run(conf, Wt.getRowPath(), initialclusters, output, measure, convergenceDelta, maxIterations, true, false);
+ KMeansDriver.run(conf,
+ Wt.getRowPath(),
+ initialclusters,
+ output,
+ measure,
+ convergenceDelta,
+ maxIterations,
+ true,
+ false);
// Read through the cluster assignments
Path clusteredPointsPath = new Path(output, "clusteredPoints");
Path inputPath = new Path(clusteredPointsPath, "part-m-00000");
int id = 0;
- for (Pair<IntWritable,WeightedVectorWritable> record :
- new SequenceFileIterable<IntWritable, WeightedVectorWritable>(inputPath, conf)) {
+ for (Pair<IntWritable,WeightedVectorWritable> record
+ : new SequenceFileIterable<IntWritable, WeightedVectorWritable>(inputPath, conf)) {
log.info("{}: {}", id++, record.getFirst().get());
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java Sun Apr 17 15:30:15 2011
@@ -345,7 +345,7 @@ public abstract class AbstractJob extend
Preconditions.checkArgument(inputOption == null || inputPath != null,
"No input specified or -Dmapred.input.dir must be provided to specify input directory");
Preconditions.checkArgument(outputOption == null || outputPath != null,
- "No output specified: or -Dmapred.output.dir must be provided to specify output directory" );
+ "No output specified: or -Dmapred.output.dir must be provided to specify output directory");
}
protected static void maybePut(Map<String, String> args, CommandLine cmdLine, Option... opt) {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileIterator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileIterator.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileIterator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileIterator.java Sun Apr 17 15:30:15 2011
@@ -19,7 +19,6 @@ package org.apache.mahout.common.iterato
import java.io.Closeable;
import java.io.IOException;
-import java.util.Iterator;
import com.google.common.collect.AbstractIterator;
import org.apache.hadoop.conf.Configuration;
@@ -33,7 +32,7 @@ import org.apache.mahout.common.IOUtils;
import org.apache.mahout.common.Pair;
/**
- * <p>{@link Iterator} over a {@link SequenceFile}'s keys and values, as a {@link Pair}
+ * <p>{@link java.util.Iterator} over a {@link SequenceFile}'s keys and values, as a {@link Pair}
* containing key and value.</p>
*/
public final class SequenceFileIterator<K extends Writable,V extends Writable>
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileValueIterator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileValueIterator.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileValueIterator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/iterator/sequencefile/SequenceFileValueIterator.java Sun Apr 17 15:30:15 2011
@@ -19,7 +19,6 @@ package org.apache.mahout.common.iterato
import java.io.Closeable;
import java.io.IOException;
-import java.util.Iterator;
import com.google.common.collect.AbstractIterator;
import org.apache.hadoop.conf.Configuration;
@@ -31,7 +30,7 @@ import org.apache.hadoop.util.Reflection
import org.apache.mahout.common.IOUtils;
/**
- * <p>{@link Iterator} over a {@link SequenceFile}'s values only.</p>
+ * <p>{@link java.util.Iterator} over a {@link SequenceFile}'s values only.</p>
*/
public final class SequenceFileValueIterator<V extends Writable> extends AbstractIterator<V> implements Closeable {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/ep/ThreadedEvolutionaryProcess.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/ep/ThreadedEvolutionaryProcess.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/ep/ThreadedEvolutionaryProcess.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/ep/ThreadedEvolutionaryProcess.java Sun Apr 17 15:30:15 2011
@@ -128,7 +128,8 @@ public class ThreadedEvolutionaryProcess
@Override
public String toString() {
- return String.format(Locale.ENGLISH, "Launched %d function evaluations\nMaximum threading width was %d", processCount, maxTask);
+ return String.format(Locale.ENGLISH,
+ "Launched %d function evaluations\nMaximum threading width was %d", processCount, maxTask);
}
public class EvalTask implements Callable<State<?, ?>> {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/PFPGrowth.java Sun Apr 17 15:30:15 2011
@@ -131,8 +131,8 @@ public final class PFPGrowth {
Path parallelCountingPath = new Path(params.get(OUTPUT), PARALLEL_COUNTING);
Path filesPattern = new Path(parallelCountingPath, FILE_PATTERN);
- for (Pair<Writable,LongWritable> record :
- new SequenceFileDirIterable<Writable,LongWritable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
+ for (Pair<Writable,LongWritable> record
+ : new SequenceFileDirIterable<Writable,LongWritable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
long value = record.getSecond().get();
if (value >= minSupport) {
queue.add(new Pair<String,Long>(record.getFirst().toString(), value));
@@ -171,9 +171,8 @@ public final class PFPGrowth {
* params should contain input and output locations as a string value, the additional parameters
* include minSupport(3), maxHeapSize(50), numGroups(1000)
*/
- public static void runPFPGrowth(Parameters params) throws IOException,
- InterruptedException,
- ClassNotFoundException {
+ public static void runPFPGrowth(Parameters params)
+ throws IOException, InterruptedException, ClassNotFoundException {
startParallelCounting(params);
startGroupingItems(params);
startTransactionSorting(params);
@@ -185,9 +184,8 @@ public final class PFPGrowth {
* Run the aggregation Job to aggregate the different TopK patterns and group each Pattern by the features
* present in it and thus calculate the final Top K frequent Patterns for each feature
*/
- public static void startAggregating(Parameters params) throws IOException,
- InterruptedException,
- ClassNotFoundException {
+ public static void startAggregating(Parameters params)
+ throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
params.set(F_LIST, "");
@@ -256,9 +254,8 @@ public final class PFPGrowth {
/**
* Count the frequencies of various features in parallel using Map/Reduce
*/
- public static void startParallelCounting(Parameters params) throws IOException,
- InterruptedException,
- ClassNotFoundException {
+ public static void startParallelCounting(Parameters params)
+ throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set(PFP_PARAMETERS, params.toString());
@@ -292,9 +289,8 @@ public final class PFPGrowth {
/**
* Run the Parallel FPGrowth Map/Reduce Job to calculate the Top K features of group dependent shards
*/
- public static void startTransactionSorting(Parameters params) throws IOException,
- InterruptedException,
- ClassNotFoundException {
+ public static void startTransactionSorting(Parameters params)
+ throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
String gList = params.get(G_LIST);
@@ -330,9 +326,8 @@ public final class PFPGrowth {
/**
* Run the Parallel FPGrowth Map/Reduce Job to calculate the Top K features of group dependent shards
*/
- public static void startParallelFPGrowth(Parameters params) throws IOException,
- InterruptedException,
- ClassNotFoundException {
+ public static void startParallelFPGrowth(Parameters params)
+ throws IOException, InterruptedException, ClassNotFoundException {
Configuration conf = new Configuration();
conf.set(PFP_PARAMETERS, params.toString());
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/ContextWriteOutputCollector.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/ContextWriteOutputCollector.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/ContextWriteOutputCollector.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/convertors/ContextWriteOutputCollector.java Sun Apr 17 15:30:15 2011
@@ -48,7 +48,7 @@ public class ContextWriteOutputCollector
@Override
public final void collect(K key, V value) throws IOException {
try {
- context.setStatus("Writing Top K patterns for: " + key.toString());
+ context.setStatus("Writing Top K patterns for: " + key);
context.write(key, value);
} catch (InterruptedException e) {
log.error("{}", e);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/fpm/pfpgrowth/fpgrowth/FPGrowth.java Sun Apr 17 15:30:15 2011
@@ -57,8 +57,8 @@ public class FPGrowth<A extends Comparab
public static List<Pair<String,TopKStringPatterns>> readFrequentPattern(Configuration conf, Path path) {
List<Pair<String,TopKStringPatterns>> ret = new ArrayList<Pair<String,TopKStringPatterns>>();
// key is feature value is count
- for (Pair<Writable,TopKStringPatterns> record :
- new SequenceFileIterable<Writable,TopKStringPatterns>(path, true, conf)) {
+ for (Pair<Writable,TopKStringPatterns> record
+ : new SequenceFileIterable<Writable,TopKStringPatterns>(path, true, conf)) {
ret.add(new Pair<String,TopKStringPatterns>(record.getFirst().toString(),
new TopKStringPatterns(record.getSecond().getPatterns())));
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TimesSquaredJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TimesSquaredJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TimesSquaredJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TimesSquaredJob.java Sun Apr 17 15:30:15 2011
@@ -183,7 +183,6 @@ public final class TimesSquaredJob {
public static Vector retrieveTimesSquaredOutputVector(Configuration conf) throws IOException {
Path outputPath = FileOutputFormat.getOutputPath(new JobConf(conf));
- FileSystem fs = FileSystem.get(conf);
Path outputFile = new Path(outputPath, "part-00000");
SequenceFileValueIterator<VectorWritable> iterator =
new SequenceFileValueIterator<VectorWritable>(outputFile, true, conf);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java Sun Apr 17 15:30:15 2011
@@ -77,12 +77,12 @@ public class DistributedLanczosSolver ex
Matrix eigenVectors,
List<Double> eigenValues,
String outputEigenVectorPathString) throws IOException {
- DistributedRowMatrix matrix =
+ DistributedRowMatrix matrix =
new DistributedRowMatrix(inputPath, outputTmpPath, numRows, numCols);
- matrix.setConf(new Configuration(originalConfig));
- setConf(originalConfig);
- solve(matrix, desiredRank, eigenVectors, eigenValues, isSymmetric);
- serializeOutput(eigenVectors, eigenValues, new Path(outputEigenVectorPathString));
+ matrix.setConf(new Configuration(originalConfig));
+ setConf(originalConfig);
+ solve(matrix, desiredRank, eigenVectors, eigenValues, isSymmetric);
+ serializeOutput(eigenVectors, eigenValues, new Path(outputEigenVectorPathString));
}
@Override
@@ -200,8 +200,8 @@ public class DistributedLanczosSolver ex
IntWritable iw = new IntWritable();
for (int i = 0; i < numEigenVectors; i++) {
// Persist eigenvectors sorted by eigenvalues in descending order
- NamedVector v = new NamedVector(eigenVectors.getRow(numEigenVectors-1-i),
- "eigenVector" + i + ", eigenvalue = " + eigenValues.get(numEigenVectors-1-i));
+ NamedVector v = new NamedVector(eigenVectors.getRow(numEigenVectors - 1 - i),
+ "eigenVector" + i + ", eigenvalue = " + eigenValues.get(numEigenVectors - 1 - i));
Writable vw = new VectorWritable(v);
iw.set(i);
seqWriter.append(iw, vw);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java Sun Apr 17 15:30:15 2011
@@ -257,8 +257,10 @@ public class EigenVerificationJob extend
for (MatrixSlice slice : eigens) {
eigenVectors.add(slice.vector());
}
- eigensToVerify = new SparseRowMatrix(new int[] { eigenVectors.size(), eigenVectors.get(0).size() }, eigenVectors
- .toArray(new Vector[eigenVectors.size()]), true, true);
+ eigensToVerify = new SparseRowMatrix(new int[] {eigenVectors.size(), eigenVectors.get(0).size()},
+ eigenVectors.toArray(new Vector[eigenVectors.size()]),
+ true,
+ true);
} else {
eigensToVerify = eigens;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/RowSimilarityJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/RowSimilarityJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/RowSimilarityJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/RowSimilarityJob.java Sun Apr 17 15:30:15 2011
@@ -266,10 +266,10 @@ public class RowSimilarityJob extends Ab
int rowB = weightedOccurrences[m].getRow();
double weightB = weightedOccurrences[m].getWeight();
double valueB = weightedOccurrences[m].getValue();
- if(rowA <= rowB){
- rowPair.set(rowA, rowB, weightA, weightB);
+ if (rowA <= rowB) {
+ rowPair.set(rowA, rowB, weightA, weightB);
} else {
- rowPair.set(rowB, rowA, weightB, weightA);
+ rowPair.set(rowB, rowA, weightB, weightA);
}
coocurrence.set(column.get(), valueA, valueB);
ctx.write(rowPair, coocurrence);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BBtJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BBtJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BBtJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BBtJob.java Sun Apr 17 15:30:15 2011
@@ -41,16 +41,15 @@ import org.apache.mahout.math.VectorWrit
* ).
*
*/
-public class BBtJob {
+public final class BBtJob {
public static final String OUTPUT_BBT = "part";
private BBtJob() {
}
- public static void run(Configuration conf, Path btPath, Path outputPath,
- int numReduceTasks) throws IOException, ClassNotFoundException,
- InterruptedException {
+ public static void run(Configuration conf, Path btPath, Path outputPath, int numReduceTasks)
+ throws IOException, ClassNotFoundException, InterruptedException {
Job job = new Job(conf);
job.setJobName("BBt-job");
@@ -73,8 +72,7 @@ public class BBtJob {
// output
job.setOutputFormatClass(SequenceFileOutputFormat.class);
FileOutputFormat.setOutputPath(job, outputPath);
- SequenceFileOutputFormat.setOutputCompressionType(job,
- CompressionType.BLOCK);
+ SequenceFileOutputFormat.setOutputCompressionType(job, CompressionType.BLOCK);
FileOutputFormat.setOutputCompressorClass(job, DefaultCodec.class);
job.getConfiguration().set("mapreduce.output.basename", OUTPUT_BBT);
@@ -98,7 +96,7 @@ public class BBtJob {
@Override
protected void map(IntWritable key, VectorWritable value, Context context)
- throws IOException, InterruptedException {
+ throws IOException, InterruptedException {
Vector btVec = value.get();
int kp = btVec.size();
if (bbtPartial == null) {
@@ -115,8 +113,7 @@ public class BBtJob {
}
@Override
- protected void cleanup(Context context) throws IOException,
- InterruptedException {
+ protected void cleanup(Context context) throws IOException, InterruptedException {
if (bbtPartial != null) {
iw.set(context.getTaskAttemptID().getTaskID().getId());
vw.set(new DenseVector(bbtPartial.getData(), true));
@@ -126,14 +123,12 @@ public class BBtJob {
}
}
- public static class BBtReducer extends
- Reducer<IntWritable, VectorWritable, IntWritable, VectorWritable> {
+ public static class BBtReducer extends Reducer<IntWritable, VectorWritable, IntWritable, VectorWritable> {
private double[] accum;
@Override
- protected void cleanup(Context context) throws IOException,
- InterruptedException {
+ protected void cleanup(Context context) throws IOException, InterruptedException {
try {
if (accum != null) {
context.write(new IntWritable(), new VectorWritable(new DenseVector(
@@ -145,8 +140,9 @@ public class BBtJob {
}
@Override
- protected void reduce(IntWritable iw, Iterable<VectorWritable> ivw,
- Context ctx) throws IOException, InterruptedException {
+ protected void reduce(IntWritable iw,
+ Iterable<VectorWritable> ivw,
+ Context ctx) throws IOException, InterruptedException {
Iterator<VectorWritable> vwIter = ivw.iterator();
Vector bbtPartial = vwIter.next().get();
if (accum == null) {
@@ -156,7 +152,7 @@ public class BBtJob {
for (int i = 0; i < accum.length; i++) {
accum[i] += bbtPartial.getQuick(i);
}
- } while (vwIter.hasNext() && null != (bbtPartial = vwIter.next().get()));
+ } while (vwIter.hasNext() && (bbtPartial = vwIter.next().get()) != null);
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BtJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BtJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BtJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/BtJob.java Sun Apr 17 15:30:15 2011
@@ -54,7 +54,7 @@ import org.apache.mahout.math.hadoop.sto
* Bt job. For details, see working notes in MAHOUT-376.
*
*/
-public class BtJob {
+public final class BtJob {
public static final String OUTPUT_Q = "Q";
public static final String OUTPUT_BT = "part";
@@ -76,7 +76,7 @@ public class BtJob {
private final VectorWritable btValue = new VectorWritable();
private int kp;
private final VectorWritable qRowValue = new VectorWritable();
- private int qCount; // debug
+ //private int qCount; // debug
void loadNextQt() throws IOException {
Writable key = new QJobKeyWritable();
@@ -114,7 +114,7 @@ public class BtJob {
// // it doesn't matter if it overflows.
// m_outputs.write( OUTPUT_Q, oKey, oV);
// }
- qCount++;
+ //qCount++;
}
@Override
@@ -151,8 +151,7 @@ public class BtJob {
// make sure Qs are inheriting A row labels.
Vector btRow = btValue.get();
- if ((aRow instanceof SequentialAccessSparseVector) ||
- (aRow instanceof RandomAccessSparseVector)) {
+ if ((aRow instanceof SequentialAccessSparseVector) || (aRow instanceof RandomAccessSparseVector)) {
for (Vector.Element el : aRow) {
double mul = el.get();
for (int j = 0; j < kp; j++) {
@@ -200,7 +199,7 @@ public class BtJob {
throw new IOException("Can't find R inputs ");
}
- Arrays.sort(rFiles, SSVDSolver.partitionComparator);
+ Arrays.sort(rFiles, SSVDSolver.PARTITION_COMPARATOR);
int block = 0;
for (FileStatus fstat : rFiles) {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/DenseBlockWritable.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/DenseBlockWritable.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/DenseBlockWritable.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/DenseBlockWritable.java Sun Apr 17 15:30:15 2011
@@ -22,21 +22,16 @@ import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.io.Writable;
-import org.apache.mahout.math.MatrixWritable;
/**
- * Ad-hoc substitution for {@link MatrixWritable}. Perhaps more useful for
+ * <p></p>Ad-hoc substitution for {@link org.apache.mahout.math.MatrixWritable}. Perhaps more useful for
* situations with mostly dense data (such as Q-blocks) but reduces GC by
- * reusing the same block memory between loads and writes.
- * <p>
+ * reusing the same block memory between loads and writes.</p>
*
- * in case of Q blocks, it doesn't even matter if they this data is dense cause
+ * <p></p>in case of Q blocks, it doesn't even matter if they this data is dense cause
* we need to unpack it into dense for fast access in computations anyway and
* even if it is not so dense the block compressor in sequence files will take
- * care of it for the serialized size.
- * <P>
- *
- *
+ * care of it for the serialized size.</p>
*/
public class DenseBlockWritable implements Writable {
private double[][] block;
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/GivensThinSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/GivensThinSolver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/GivensThinSolver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/GivensThinSolver.java Sun Apr 17 15:30:15 2011
@@ -548,7 +548,7 @@ public class GivensThinSolver {
}
- private static class TriangularRowView extends AbstractVector {
+ private static final class TriangularRowView extends AbstractVector {
private final UpperTriangular viewed;
private int rowNum;
@@ -605,7 +605,7 @@ public class GivensThinSolver {
}
@Override
- protected Matrix matrixLike(int rows, int columns) {
+ public Matrix matrixLike(int rows, int columns) {
throw new UnsupportedOperationException();
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/Omega.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/Omega.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/Omega.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/Omega.java Sun Apr 17 15:30:15 2011
@@ -61,8 +61,7 @@ public class Omega {
assert yRow.length == kp;
Arrays.fill(yRow, 0);
- if ((aRow instanceof SequentialAccessSparseVector)||
- (aRow instanceof RandomAccessSparseVector)){
+ if ((aRow instanceof SequentialAccessSparseVector) || (aRow instanceof RandomAccessSparseVector)) {
int j = 0;
for (Element el : aRow) {
accumDots(j, el.get(), yRow);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/QJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/QJob.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/QJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/QJob.java Sun Apr 17 15:30:15 2011
@@ -56,7 +56,7 @@ import org.apache.mahout.math.VectorWrit
*
*
*/
-public class QJob {
+public final class QJob {
public static final String PROP_OMEGA_SEED = "ssvd.omegaseed";
public static final String PROP_K = "ssvd.k";
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDCli.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDCli.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDCli.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDCli.java Sun Apr 17 15:30:15 2011
@@ -35,8 +35,6 @@ import org.apache.mahout.math.VectorWrit
/**
* Mahout CLI adapter for SSVDSolver
- *
- *
*/
public class SSVDCli extends AbstractJob {
@@ -82,8 +80,7 @@ public class SSVDCli extends AbstractJob
throw new IOException("No Hadoop configuration present");
}
- SSVDSolver solver = new SSVDSolver(conf, new Path[] { new Path(input) },
- new Path(tempDir), r, k, p, reduceTasks);
+ SSVDSolver solver = new SSVDSolver(conf, new Path[] {new Path(input)}, new Path(tempDir), r, k, p, reduceTasks);
solver.setMinSplitSize(minSplitSize);
solver.setComputeU(computeU);
solver.setComputeV(computeV);
@@ -99,8 +96,8 @@ public class SSVDCli extends AbstractJob
Path outPath = new Path(output);
fs.mkdirs(outPath);
- SequenceFile.Writer sigmaW = SequenceFile.createWriter(fs, conf, new Path(
- outPath, "sigma"), NullWritable.class, VectorWritable.class);
+ SequenceFile.Writer sigmaW =
+ SequenceFile.createWriter(fs, conf, new Path(outPath, "sigma"), NullWritable.class, VectorWritable.class);
try {
Writable sValues = new VectorWritable(new DenseVector(
Arrays.copyOf(solver.getSingularValues(), k), true));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/stochasticsvd/SSVDSolver.java Sun Apr 17 15:30:15 2011
@@ -382,7 +382,7 @@ public class SSVDSolver {
private static final Pattern OUTPUT_FILE_PATTERN = Pattern.compile("(\\w+)-(m|r)-(\\d+)(\\.\\w+)?");
- static final Comparator<FileStatus> partitionComparator = new Comparator<FileStatus>() {
+ static final Comparator<FileStatus> PARTITION_COMPARATOR = new Comparator<FileStatus>() {
private final Matcher matcher = OUTPUT_FILE_PATTERN.matcher("");
@Override
@@ -430,7 +430,7 @@ public class SSVDSolver {
// assume it is partitioned output, so we need to read them up
// in order of partitions.
- Arrays.sort(files, partitionComparator);
+ Arrays.sort(files, PARTITION_COMPARATOR);
for (FileStatus fstat : files) {
for (VectorWritable value : new SequenceFileValueIterable<VectorWritable>(fstat.getPath(), true, conf)) {
@@ -449,7 +449,8 @@ public class SSVDSolver {
}
public static UpperTriangular loadUpperTriangularMatrix(FileSystem fs,
- Path glob, Configuration conf) throws IOException {
+ Path glob,
+ Configuration conf) throws IOException {
FileStatus[] files = fs.globStatus(glob);
if (files == null) {
@@ -458,7 +459,7 @@ public class SSVDSolver {
// assume it is partitioned output, so we need to read them up
// in order of partitions.
- Arrays.sort(files, partitionComparator);
+ Arrays.sort(files, PARTITION_COMPARATOR);
UpperTriangular result = null;
for (FileStatus fstat : files) {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/DictionaryVectorizer.java Sun Apr 17 15:30:15 2011
@@ -210,8 +210,8 @@ public final class DictionaryVectorizer
long currentChunkSize = 0;
Path filesPattern = new Path(wordCountPath, OUTPUT_FILES_PATTERN);
int i = 0;
- for (Pair<Writable,Writable> record :
- new SequenceFileDirIterable<Writable,Writable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
+ for (Pair<Writable,Writable> record
+ : new SequenceFileDirIterable<Writable,Writable>(filesPattern, PathType.GLOB, null, null, true, conf)) {
if (currentChunkSize > chunkSizeLimit) {
dictWriter.close();
chunkIndex++;
@@ -279,7 +279,7 @@ public final class DictionaryVectorizer
Job job = new Job(conf);
job.setJobName("DictionaryVectorizer::MakePartialVectors: input-folder: " + input
- + ", dictionary-file: " + dictionaryFilePath.toString());
+ + ", dictionary-file: " + dictionaryFilePath);
job.setJarByClass(DictionaryVectorizer.class);
job.setMapOutputKeyClass(Text.class);
@@ -316,7 +316,7 @@ public final class DictionaryVectorizer
Job job = new Job(conf);
- job.setJobName("DictionaryVectorizer::WordCount: input-folder: " + input.toString());
+ job.setJobName("DictionaryVectorizer::WordCount: input-folder: " + input);
job.setJarByClass(DictionaryVectorizer.class);
job.setOutputKeyClass(Text.class);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/SparseVectorsFromSequenceFiles.java Sun Apr 17 15:30:15 2011
@@ -135,7 +135,8 @@ public final class SparseVectorsFromSequ
.withOption(chunkSizeOpt).withOption(outputDirOpt).withOption(inputDirOpt).withOption(minDFOpt)
.withOption(maxDFPercentOpt).withOption(weightOpt).withOption(powerOpt).withOption(minLLROpt)
.withOption(numReduceTasksOpt).withOption(maxNGramSizeOpt).withOption(overwriteOutput)
- .withOption(helpOpt).withOption(sequentialAccessVectorOpt).withOption(namedVectorOpt).withOption(logNormalizeOpt)
+ .withOption(helpOpt).withOption(sequentialAccessVectorOpt).withOption(namedVectorOpt)
+ .withOption(logNormalizeOpt)
.create();
try {
Parser parser = new Parser();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/CollocMapper.java Sun Apr 17 15:30:15 2011
@@ -109,67 +109,57 @@ public class CollocMapper extends Mapper
}
} while (sf.incrementToken());
- try {
- final GramKey gramKey = new GramKey();
+ final GramKey gramKey = new GramKey();
- ngrams.forEachPair(new ObjectIntProcedure<String>() {
- @Override
- public boolean apply(String term, int frequency) {
- // obtain components, the leading (n-1)gram and the trailing unigram.
- int i = term.lastIndexOf(' '); // TODO: fix for non-whitespace delimited languages.
- if (i != -1) { // bigram, trigram etc
-
- try {
- Gram ngram = new Gram(term, frequency, Gram.Type.NGRAM);
- Gram head = new Gram(term.substring(0, i), frequency, Gram.Type.HEAD);
- Gram tail = new Gram(term.substring(i + 1), frequency, Gram.Type.TAIL);
-
- gramKey.set(head, EMPTY);
- context.write(gramKey, head);
-
- gramKey.set(head, ngram.getBytes());
- context.write(gramKey, ngram);
-
- gramKey.set(tail, EMPTY);
- context.write(gramKey, tail);
-
- gramKey.set(tail, ngram.getBytes());
- context.write(gramKey, ngram);
-
- } catch (IOException e) {
- throw new IllegalStateException(e);
- } catch (InterruptedException e) {
- throw new IllegalStateException(e);
- }
- }
- return true;
- }
- });
+ ngrams.forEachPair(new ObjectIntProcedure<String>() {
+ @Override
+ public boolean apply(String term, int frequency) {
+ // obtain components, the leading (n-1)gram and the trailing unigram.
+ int i = term.lastIndexOf(' '); // TODO: fix for non-whitespace delimited languages.
+ if (i != -1) { // bigram, trigram etc
- unigrams.forEachPair(new ObjectIntProcedure<String>() {
- @Override
- public boolean apply(String term, int frequency) {
try {
- Gram unigram = new Gram(term, frequency, Gram.Type.UNIGRAM);
- gramKey.set(unigram, EMPTY);
- context.write(gramKey, unigram);
+ Gram ngram = new Gram(term, frequency, Gram.Type.NGRAM);
+ Gram head = new Gram(term.substring(0, i), frequency, Gram.Type.HEAD);
+ Gram tail = new Gram(term.substring(i + 1), frequency, Gram.Type.TAIL);
+
+ gramKey.set(head, EMPTY);
+ context.write(gramKey, head);
+
+ gramKey.set(head, ngram.getBytes());
+ context.write(gramKey, ngram);
+
+ gramKey.set(tail, EMPTY);
+ context.write(gramKey, tail);
+
+ gramKey.set(tail, ngram.getBytes());
+ context.write(gramKey, ngram);
+
} catch (IOException e) {
throw new IllegalStateException(e);
} catch (InterruptedException e) {
throw new IllegalStateException(e);
}
- return true;
}
- });
- } catch (IllegalStateException ise) {
- // catch an re-throw original exceptions from the procedures.
- if (ise.getCause() instanceof IOException) {
- throw (IOException) ise.getCause();
- } else {
- // wasn't what was expected, so re-throw
- throw ise;
+ return true;
}
- }
+ });
+
+ unigrams.forEachPair(new ObjectIntProcedure<String>() {
+ @Override
+ public boolean apply(String term, int frequency) {
+ try {
+ Gram unigram = new Gram(term, frequency, Gram.Type.UNIGRAM);
+ gramKey.set(unigram, EMPTY);
+ context.write(gramKey, unigram);
+ } catch (IOException e) {
+ throw new IllegalStateException(e);
+ } catch (InterruptedException e) {
+ throw new IllegalStateException(e);
+ }
+ return true;
+ }
+ });
context.getCounter(Count.NGRAM_TOTAL).increment(count);
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/collocations/llr/Gram.java Sun Apr 17 15:30:15 2011
@@ -139,7 +139,7 @@ public class Gram extends BinaryComparab
try {
return Text.decode(bytes, 1, length - 1);
} catch (CharacterCodingException e) {
- throw new IllegalStateException("Should not have happened " + e.toString());
+ throw new IllegalStateException("Should not have happened " + e);
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TFPartialVectorReducer.java Sun Apr 17 15:30:15 2011
@@ -123,8 +123,8 @@ public class TFPartialVectorReducer exte
Path dictionaryFile = new Path(localFiles[0].getPath());
// key is word value is id
- for (Pair<Writable,IntWritable> record :
- new SequenceFileIterable<Writable,IntWritable>(dictionaryFile, true, conf)) {
+ for (Pair<Writable,IntWritable> record
+ : new SequenceFileIterable<Writable,IntWritable>(dictionaryFile, true, conf)) {
dictionary.put(record.getFirst().toString(), record.getSecond().get());
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TermCountReducer.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TermCountReducer.java?rev=1094158&r1=1094157&r2=1094158&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TermCountReducer.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/vectorizer/term/TermCountReducer.java Sun Apr 17 15:30:15 2011
@@ -32,7 +32,8 @@ public class TermCountReducer extends Re
private int minSupport;
@Override
- protected void reduce(Text key, Iterable<LongWritable> values, Context context) throws IOException, InterruptedException {
+ protected void reduce(Text key, Iterable<LongWritable> values, Context context)
+ throws IOException, InterruptedException {
long sum = 0;
for (LongWritable value : values) {
sum += value.get();
@@ -45,7 +46,8 @@ public class TermCountReducer extends Re
@Override
protected void setup(Context context) throws IOException, InterruptedException {
super.setup(context);
- minSupport = context.getConfiguration().getInt(DictionaryVectorizer.MIN_SUPPORT, DictionaryVectorizer.DEFAULT_MIN_SUPPORT);
+ minSupport = context.getConfiguration().getInt(DictionaryVectorizer.MIN_SUPPORT,
+ DictionaryVectorizer.DEFAULT_MIN_SUPPORT);
}
}