You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2008/10/24 13:47:45 UTC
svn commit: r707606 [1/4] - in /lucene/mahout/trunk: ./
core/src/main/java/org/apache/mahout/cf/taste/hadoop/
core/src/main/java/org/apache/mahout/cf/taste/impl/common/
core/src/main/java/org/apache/mahout/cf/taste/impl/model/
core/src/main/java/org/ap...
Author: srowen
Date: Fri Oct 24 04:47:42 2008
New Revision: 707606
URL: http://svn.apache.org/viewvc?rev=707606&view=rev
Log:
mahout-88 patch submission
Removed:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/UpdatableFloat.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/UpdatableLong.java
Modified:
lucene/mahout/trunk/common-build.xml
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/ByItemIDComparator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FastMap.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItem.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUser.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/AveragingPreferenceInferrer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/Counters.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/InverseUserFrequency.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ClassifierResult.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesNormalizedWeightReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaNormalizerReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesThetaReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyCombiner.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/CanopyReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/canopy/ClusterMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansDriver.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/FuzzyKMeansMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/fuzzykmeans/SoftCluster.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/kmeans/KMeansMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopy.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyCombiner.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/clustering/meanshift/MeanShiftCanopyReducer.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Classifier.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/Model.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/EvalMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutEvaluator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/MahoutFitnessEvaluator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/OutputUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/STEvolutionEngine.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/ga/watchmaker/STFitnessEvaluator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/AbstractMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/AbstractVector.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/CardinalityException.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/DenseMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/DenseVector.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/IndexException.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/Matrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/MatrixView.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/SparseColumnMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/SparseMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/SparseRowMatrix.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/SparseVector.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/Vector.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/matrix/VectorView.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/DistanceMeasure.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/EuclideanDistanceMeasure.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/ManhattanDistanceMeasure.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/StringUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/TanimotoDistanceMeasure.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedEuclideanDistanceMeasure.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/WeightedManhattanDistanceMeasure.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/parameters/DoubleParameter.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/utils/parameters/IntegerParameter.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/BayesClassifierTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/classifier/bayes/CBayesClassifierTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/fuzzykmeans/TestFuzzyKmeansClustering.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/clustering/meanshift/TestMeanShift.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/EvalMapperTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/MahoutEvaluatorTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/STFitnessEvaluatorTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/ga/watchmaker/utils/DummyEvaluator.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/matrix/VectorTest.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/XmlInputFormat.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/canopy/InputMapper.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/clustering/syntheticcontrol/meanshift/InputMapper.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/cd/CDCrossover.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/cd/CDFitness.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/cd/CDFitnessEvaluator.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/cd/tool/DescriptionUtils.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/cd/tool/ToolCombiner.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/cd/tool/ToolReducer.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/BruteForceTravellingSalesman.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/EuropeanDistanceLookup.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/EvolutionaryTravellingSalesman.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/ExecutionPanel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/ItineraryPanel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/StrategyPanel.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/ga/watchmaker/travellingsalesman/TravellingSalesman.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/CDFitnessTest.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/CDMutationTest.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/CDRuleTest.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/tool/CDInfosToolTest.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/tool/DescriptionUtilsTest.java
lucene/mahout/trunk/examples/src/test/java/org/apache/mahout/ga/watchmaker/cd/utils/MockDataSet.java
Modified: lucene/mahout/trunk/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/common-build.xml?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/common-build.xml (original)
+++ lucene/mahout/trunk/common-build.xml Fri Oct 24 04:47:42 2008
@@ -107,6 +107,7 @@
encoding="utf8"
sourcepath=""
classpathref="@{classpathref}">
+ <compilerarg value="-Xlint:all,-serial"/>
<nested />
</javac>
</sequential>
@@ -211,4 +212,4 @@
</manifest>
</target>
-</project>
\ No newline at end of file
+</project>
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/ByItemIDComparator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/ByItemIDComparator.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/ByItemIDComparator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/ByItemIDComparator.java Fri Oct 24 04:47:42 2008
@@ -22,9 +22,9 @@
final class ByItemIDComparator implements Comparator<ItemPrefWritable>, Serializable {
- private static final Comparator instance = new ByItemIDComparator();
+ private static final Comparator<ItemPrefWritable> instance = new ByItemIDComparator();
- public static Comparator getInstance() {
+ public static Comparator<ItemPrefWritable> getInstance() {
return instance;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesJob.java Fri Oct 24 04:47:42 2008
@@ -19,7 +19,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
@@ -59,11 +59,11 @@
jobConf.setMapperClass(IdentityMapper.class);
jobConf.setMapOutputKeyClass(ItemItemWritable.class);
- jobConf.setMapOutputValueClass(FloatWritable.class);
+ jobConf.setMapOutputValueClass(DoubleWritable.class);
jobConf.setReducerClass(SlopeOneDiffsToAveragesReducer.class);
jobConf.setOutputKeyClass(ItemItemWritable.class);
- jobConf.setOutputValueClass(FloatWritable.class);
+ jobConf.setOutputValueClass(DoubleWritable.class);
jobConf.setOutputFormat(TextOutputFormat.class);
FileOutputFormat.setOutputPath(jobConf, outputPathPath);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOneDiffsToAveragesReducer.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.cf.taste.hadoop;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
@@ -28,19 +28,19 @@
public final class SlopeOneDiffsToAveragesReducer
extends MapReduceBase
- implements Reducer<ItemItemWritable, FloatWritable, ItemItemWritable, FloatWritable> {
+ implements Reducer<ItemItemWritable, DoubleWritable, ItemItemWritable, DoubleWritable> {
public void reduce(ItemItemWritable key,
- Iterator<FloatWritable> values,
- OutputCollector<ItemItemWritable, FloatWritable> output,
+ Iterator<DoubleWritable> values,
+ OutputCollector<ItemItemWritable, DoubleWritable> output,
Reporter reporter) throws IOException {
int count = 0;
- float total = 0.0f;
+ double total = 0.0;
while (values.hasNext()) {
total += values.next().get();
count++;
}
- output.collect(key, new FloatWritable(total / count));
+ output.collect(key, new DoubleWritable((total / count)));
}
}
\ No newline at end of file
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsJob.java Fri Oct 24 04:47:42 2008
@@ -19,9 +19,9 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
@@ -64,7 +64,7 @@
jobConf.setReducerClass(SlopeOnePrefsToDiffsReducer.class);
jobConf.setOutputKeyClass(ItemItemWritable.class);
- jobConf.setOutputValueClass(FloatWritable.class);
+ jobConf.setOutputValueClass(DoubleWritable.class);
jobConf.setOutputFormat(SequenceFileOutputFormat.class);
SequenceFileOutputFormat.setOutputCompressionType(jobConf, SequenceFile.CompressionType.RECORD);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/SlopeOnePrefsToDiffsReducer.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.cf.taste.hadoop;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
@@ -27,24 +27,23 @@
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
-/**
- */
public final class SlopeOnePrefsToDiffsReducer
extends MapReduceBase
- implements Reducer<Text, ItemPrefWritable, ItemItemWritable, FloatWritable> {
+ implements Reducer<Text, ItemPrefWritable, ItemItemWritable, DoubleWritable> {
public void reduce(Text key,
Iterator<ItemPrefWritable> values,
- OutputCollector<ItemItemWritable, FloatWritable> output,
+ OutputCollector<ItemItemWritable, DoubleWritable> output,
Reporter reporter) throws IOException {
List<ItemPrefWritable> prefs = new ArrayList<ItemPrefWritable>();
while (values.hasNext()) {
prefs.add(new ItemPrefWritable(values.next()));
}
- Collections.sort(prefs, ByItemIDComparator.getInstance());
+ Collections.sort(prefs, (Comparator<? super ItemPrefWritable>) ByItemIDComparator.getInstance());
int size = prefs.size();
for (int i = 0; i < size; i++) {
ItemPrefWritable first = prefs.get(i);
@@ -54,7 +53,7 @@
ItemPrefWritable second = prefs.get(j);
String itemBID = second.getItemID();
double itemBValue = second.getPrefValue();
- output.collect(new ItemItemWritable(itemAID, itemBID), new FloatWritable((float) (itemBValue - itemAValue)));
+ output.collect(new ItemItemWritable(itemAID, itemBID), new DoubleWritable(itemBValue - itemAValue));
}
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FastMap.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FastMap.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FastMap.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/FastMap.java Fri Oct 24 04:47:42 2008
@@ -81,6 +81,7 @@
* @param maxSize max capacity
* @throws IllegalArgumentException if size is less than 1 or at least half of {@link #MAX_INT_SMALLER_TWIN_PRIME}
*/
+ @SuppressWarnings("unchecked")
public FastMap(int size, int maxSize) throws IllegalArgumentException {
if (size < 1) {
throw new IllegalArgumentException("size must be at least 1");
@@ -292,6 +293,7 @@
rehash(nextTwinPrime(2 * hashSize));
}
+ @SuppressWarnings("unchecked")
private void rehash(int newHashSize) {
K[] oldKeys = keys;
V[] oldValues = values;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java Fri Oct 24 04:47:42 2008
@@ -60,6 +60,7 @@
*
* @param users {@link User}s to include in this {@link GenericDataModel}
*/
+ @SuppressWarnings("unchecked")
public GenericDataModel(Iterable<? extends User> users) {
if (users == null) {
throw new IllegalArgumentException("users is null");
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItem.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItem.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItem.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItem.java Fri Oct 24 04:47:42 2008
@@ -64,6 +64,7 @@
return "Item[id:" + String.valueOf(id) + ']';
}
+ @SuppressWarnings("unchecked")
public int compareTo(Item item) {
return id.compareTo((K) item.getID());
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUser.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUser.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUser.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUser.java Fri Oct 24 04:47:42 2008
@@ -98,6 +98,7 @@
return "User[id:" + String.valueOf(id) + ']';
}
+ @SuppressWarnings("unchecked")
public int compareTo(User o) {
return id.compareTo((K) o.getID());
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/AveragingPreferenceInferrer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/AveragingPreferenceInferrer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/AveragingPreferenceInferrer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/AveragingPreferenceInferrer.java Fri Oct 24 04:47:42 2008
@@ -39,7 +39,7 @@
public final class AveragingPreferenceInferrer implements PreferenceInferrer {
private static final Retriever<User, Double> RETRIEVER = new PrefRetriever();
- private static final Double ZERO = new Double(0.0);
+ private static final Double ZERO = 0.0;
private final Cache<User, Double> averagePreferenceValue;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/Counters.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/Counters.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/Counters.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/Counters.java Fri Oct 24 04:47:42 2008
@@ -26,29 +26,29 @@
*/
final class Counters<T> {
- private final Map<T, MutableInteger> counts = new FastMap<T, MutableInteger>();
+ private final Map<T, int[]> counts = new FastMap<T, int[]>();
void increment(T key) {
- MutableInteger count = counts.get(key);
+ int[] count = counts.get(key);
if (count == null) {
- MutableInteger newCount = new MutableInteger();
- newCount.value = 1;
+ int[] newCount = new int[1];
+ newCount[0] = 1;
counts.put(key, newCount);
} else {
- count.value++;
+ count[0]++;
}
}
int getCount(T key) {
- MutableInteger count = counts.get(key);
- return count == null ? 0 : count.value;
+ int[] count = counts.get(key);
+ return count == null ? 0 : count[0];
}
int size() {
return counts.size();
}
- Iterable<Map.Entry<T, MutableInteger>> getEntrySet() {
+ Iterable<Map.Entry<T, int[]>> getEntrySet() {
return counts.entrySet();
}
@@ -57,16 +57,4 @@
return "Counters[" + counts + ']';
}
- static final class MutableInteger {
-
- // This is intentionally package-private in order to allow access from the containing Counters class
- // without making the compiler generate a synthetic accessor
- int value;
-
- @Override
- public String toString() {
- return "MutableInteger[" + value + ']';
- }
- }
-
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/InverseUserFrequency.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/InverseUserFrequency.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/InverseUserFrequency.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/transforms/InverseUserFrequency.java Fri Oct 24 04:47:42 2008
@@ -110,9 +110,9 @@
}
Map<Item, Double> newIufFactors = new FastMap<Item, Double>(itemPreferenceCounts.size());
double logFactor = Math.log(logBase);
- for (Map.Entry<Item, Counters.MutableInteger> entry : itemPreferenceCounts.getEntrySet()) {
+ for (Map.Entry<Item, int[]> entry : itemPreferenceCounts.getEntrySet()) {
newIufFactors.put(entry.getKey(),
- Math.log((double) numUsers / (double) entry.getValue().value) / logFactor);
+ Math.log((double) numUsers / (double) entry.getValue()[0]) / logFactor);
}
iufFactors.set(Collections.unmodifiableMap(newIufFactors));
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ClassifierResult.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ClassifierResult.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ClassifierResult.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ClassifierResult.java Fri Oct 24 04:47:42 2008
@@ -22,12 +22,12 @@
*/
public class ClassifierResult {
private String label;
- private float score;
+ private double score;
public ClassifierResult() {
}
- public ClassifierResult(String label, float score) {
+ public ClassifierResult(String label, double score) {
this.label = label;
this.score = score;
}
@@ -40,7 +40,7 @@
return label;
}
- public float getScore() {
+ public double getScore() {
return score;
}
@@ -48,7 +48,7 @@
this.label = label;
}
- public void setScore(float score) {
+ public void setScore(double score) {
this.score = score;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/ConfusionMatrix.java Fri Oct 24 04:47:42 2008
@@ -44,15 +44,16 @@
return labels;
}
- public float getAccuracy(String label){
+ public double getAccuracy(String label){
int labelId = labelMap.get(label);
int labelTotal = 0;
int correct = 0;
for(int i = 0 ;i < labels.size() ;i++){
- labelTotal+= confusionMatrix[labelId][i];
- if(i==labelId) correct = confusionMatrix[labelId][i];
+ labelTotal += confusionMatrix[labelId][i];
+ if(i == labelId)
+ correct = confusionMatrix[labelId][i];
}
- return (float)100 * correct / labelTotal;
+ return 100.0 * correct / labelTotal;
}
public int getCorrect(String label){
@@ -60,10 +61,10 @@
return confusionMatrix[labelId][labelId];
}
- public float getTotal(String label){
+ public double getTotal(String label){
int labelId = labelMap.get(label);
int labelTotal = 0;
- for(int i = 0 ;i < labels.size() ;i++){
+ for (int i = 0 ;i < labels.size() ;i++){
labelTotal+= confusionMatrix[labelId][i];
}
return labelTotal;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesClassifier.java Fri Oct 24 04:47:42 2008
@@ -40,7 +40,8 @@
* @param model The model
* @param document The document to classify
* @param defaultCategory The default category to assign
- * @param numResults The maximum number of results to return, ranked by score. Ties are broken by comparing the category
+ * @param numResults The maximum number of results to return, ranked by score.
+ * Ties are broken by comparing the category
* @return A Collection of {@link org.apache.mahout.classifier.ClassifierResult}s.
*/
public Collection<ClassifierResult> classify(Model model, String[] document, String defaultCategory, int numResults) {
@@ -49,8 +50,8 @@
PriorityQueue pq = new ClassifierResultPriorityQueue(numResults);
ClassifierResult tmp;
for (String category : categories){
- float prob = documentProbability(model, category, document);
- if (prob < 0) {
+ double prob = documentProbability(model, category, document);
+ if (prob < 0.0) {
tmp = new ClassifierResult(category, prob);
pq.insert(tmp);
}
@@ -76,12 +77,11 @@
*/
public ClassifierResult classify(Model model, String[] document, String defaultCategory) {
ClassifierResult result = new ClassifierResult(defaultCategory);
- float min = 0.0f;
+ double min = 0.0;
Collection<String> categories = model.getLabels();
for (String category : categories) {
- float prob = documentProbability(model, category, document);
-
+ double prob = documentProbability(model, category, document);
if (prob < min) {
min = prob;
result.setLabel(category);
@@ -92,7 +92,8 @@
}
/**
- * Calculate the document probability as the multiplication of the {@link org.apache.mahout.common.Model#FeatureWeight(String, String)} for each word given the label
+ * Calculate the document probability as the multiplication of the
+ * {@link org.apache.mahout.common.Model#FeatureWeight(String, String)} for each word given the label
*
* @param model The {@link org.apache.mahout.common.Model}
* @param label The label to calculate the probability of
@@ -100,8 +101,8 @@
* @return The probability
* @see Model#FeatureWeight(String, String)
*/
- public float documentProbability(Model model, String label, String[] document) {
- float result = 0.0f;
+ public double documentProbability(Model model, String label, String[] document) {
+ double result = 0.0;
Map<String, Integer> wordList = new HashMap<String, Integer>(1000);
for (String word : document) {
if (wordList.containsKey(word)) {
@@ -130,8 +131,8 @@
ClassifierResult cr1 = (ClassifierResult) a;
ClassifierResult cr2 = (ClassifierResult) b;
- float score1 = cr1.getScore();
- float score2 = cr2.getScore();
+ double score1 = cr1.getScore();
+ double score2 = cr2.getScore();
return score1 == score2 ? cr1.getLabel().compareTo(cr2.getLabel()) < 0 : score1 < score2;
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesModel.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesModel.java Fri Oct 24 04:47:42 2008
@@ -28,55 +28,55 @@
private static final Logger log = LoggerFactory.getLogger(BayesModel.class);
@Override
- protected float getWeight(Integer label, Integer feature) {
- float result = 0.0f;
- Map<Integer, Float> featureWeights = featureLabelWeights.get(feature);
+ protected double getWeight(Integer label, Integer feature) {
+ double result = 0.0;
+ Map<Integer,Double> featureWeights = featureLabelWeights.get(feature);
if (featureWeights.containsKey(label)) {
result = featureWeights.get(label).floatValue();
}
- float vocabCount = featureList.size();
- float sumLabelWeight = getSumLabelWeight(label);
+ double vocabCount = featureList.size();
+ double sumLabelWeight = getSumLabelWeight(label);
- float numerator = result + alpha_i;
- float denominator =(sumLabelWeight + vocabCount);
+ double numerator = result + alpha_i;
+ double denominator =(sumLabelWeight + vocabCount);
- float weight = (float) Math.log(numerator /denominator);
- result = (-1.0f * (weight));
+ double weight = Math.log(numerator /denominator);
+ result = -weight;
return result;
}
@Override
- protected float getWeightUnprocessed(Integer label, Integer feature) {
- float result;
- Map<Integer, Float> featureWeights = featureLabelWeights.get(feature);
+ protected double getWeightUnprocessed(Integer label, Integer feature) {
+ double result;
+ Map<Integer,Double> featureWeights = featureLabelWeights.get(feature);
if (featureWeights.containsKey(label)) {
result = featureWeights.get(label);
} else {
- result = 0.0f;
+ result = 0.0;
}
return result;
}
@Override
public void InitializeNormalizer() {
- float perLabelWeightSumNormalisationFactor = Float.MAX_VALUE;
+ double perLabelWeightSumNormalisationFactor = Double.MAX_VALUE;
log.info("{}", thetaNormalizer);
for (Integer label : thetaNormalizer.keySet()) {
- float Sigma_W_ij = thetaNormalizer.get(label);
+ double Sigma_W_ij = thetaNormalizer.get(label);
if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
}
}
for (Integer label : thetaNormalizer.keySet()) {
- float Sigma_W_ij = thetaNormalizer.get(label);
+ double Sigma_W_ij = thetaNormalizer.get(label);
thetaNormalizer.put(label, Sigma_W_ij
/ perLabelWeightSumNormalisationFactor);
}
@@ -85,26 +85,26 @@
@Override
public void GenerateModel() {
- float vocabCount = featureList.size();
+ double vocabCount = featureList.size();
- float[] perLabelThetaNormalizer = new float[labelList.size()];
+ double[] perLabelThetaNormalizer = new double[labelList.size()];
- float perLabelWeightSumNormalisationFactor = Float.MAX_VALUE;
+ double perLabelWeightSumNormalisationFactor = Double.MAX_VALUE;
for (int feature = 0, maxFeatures = featureList.size(); feature < maxFeatures; feature++) {
Integer featureInt = feature;
for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
Integer labelInt = label;
- float D_ij = getWeightUnprocessed(labelInt, featureInt);
- float sumLabelWeight = getSumLabelWeight(labelInt);
+ double D_ij = getWeightUnprocessed(labelInt, featureInt);
+ double sumLabelWeight = getSumLabelWeight(labelInt);
// TODO srowen says sigma_j is unused
- //float sigma_j = getSumFeatureWeight(featureInt);
+ //double sigma_j = getSumFeatureWeight(featureInt);
- float numerator = D_ij + alpha_i;
- float denominator = sumLabelWeight + vocabCount;
+ double numerator = D_ij + alpha_i;
+ double denominator = sumLabelWeight + vocabCount;
- float weight = (float) Math.log(numerator / denominator);
+ double weight = Math.log(numerator / denominator);
if (D_ij != 0)
setWeight(labelInt, featureInt, weight);
@@ -115,14 +115,14 @@
}
log.info("Normalizing Weights");
for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- float Sigma_W_ij = perLabelThetaNormalizer[label];
+ double Sigma_W_ij = perLabelThetaNormalizer[label];
if (perLabelWeightSumNormalisationFactor > Math.abs(Sigma_W_ij)) {
perLabelWeightSumNormalisationFactor = Math.abs(Sigma_W_ij);
}
}
for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
- float Sigma_W_ij = perLabelThetaNormalizer[label];
+ double Sigma_W_ij = perLabelThetaNormalizer[label];
perLabelThetaNormalizer[label] = Sigma_W_ij
/ perLabelWeightSumNormalisationFactor;
}
@@ -131,18 +131,17 @@
Integer featureInt = feature;
for (int label = 0, maxLabels = labelList.size(); label < maxLabels; label++) {
Integer labelInt = label;
- float W_ij = getWeightUnprocessed(labelInt, featureInt);
+ double W_ij = getWeightUnprocessed(labelInt, featureInt);
if (W_ij == 0)
continue;
- float Sigma_W_ij = perLabelThetaNormalizer[label];
- float normalizedWeight = -1.0f * (W_ij / Sigma_W_ij);
+ double Sigma_W_ij = perLabelThetaNormalizer[label];
+ double normalizedWeight = -W_ij / Sigma_W_ij;
setWeight(labelInt, featureInt, normalizedWeight);
}
}
}
-
/**
* Get the weighted probability of the feature.
*
@@ -151,7 +150,7 @@
* @return The weighted probability
*/
@Override
- public float FeatureWeight(Integer label, Integer feature) {
+ public double FeatureWeight(Integer label, Integer feature) {
return getWeight(label, feature);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerDriver.java Fri Oct 24 04:47:42 2008
@@ -20,7 +20,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DefaultStringifier;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -31,7 +31,6 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
import java.util.Map;
import java.io.IOException;
@@ -70,7 +69,7 @@
conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(FloatWritable.class);
+ conf.setOutputValueClass(DoubleWritable.class);
SequenceFileInputFormat.addInputPath(conf, new Path(output + "/trainer-tfIdf/trainer-tfIdf"));
Path outPath = new Path(output + "/trainer-thetaNormalizer");
SequenceFileOutputFormat.setOutputPath(conf, outPath);
@@ -92,34 +91,34 @@
SequenceFileModelReader reader = new SequenceFileModelReader();
Path Sigma_kFiles = new Path(output+"/trainer-weights/Sigma_k/*");
- Map<String,Float> labelWeightSum= reader.readLabelSums(dfs, Sigma_kFiles, conf);
- DefaultStringifier<Map<String,Float>> mapStringifier =
- new DefaultStringifier<Map<String,Float>>(conf, GenericsUtil.getClass(labelWeightSum));
+ Map<String,Double> labelWeightSum = reader.readLabelSums(dfs, Sigma_kFiles, conf);
+ DefaultStringifier<Map<String,Double>> mapStringifier =
+ new DefaultStringifier<Map<String,Double>>(conf, GenericsUtil.getClass(labelWeightSum));
String labelWeightSumString = mapStringifier.toString(labelWeightSum);
log.info("Sigma_k for Each Label");
- Map<String,Float> c = mapStringifier.fromString(labelWeightSumString);
+ Map<String,Double> c = mapStringifier.fromString(labelWeightSumString);
log.info("{}", c);
conf.set("cnaivebayes.sigma_k", labelWeightSumString);
Path sigma_kSigma_jFile = new Path(output+"/trainer-weights/Sigma_kSigma_j/*");
- Float sigma_jSigma_k = reader.readSigma_jSigma_k(dfs, sigma_kSigma_jFile, conf);
- DefaultStringifier<Float> floatStringifier = new DefaultStringifier<Float>(conf, Float.class);
- String sigma_jSigma_kString = floatStringifier.toString(sigma_jSigma_k);
+ double sigma_jSigma_k = reader.readSigma_jSigma_k(dfs, sigma_kSigma_jFile, conf);
+ DefaultStringifier<Double> stringifier = new DefaultStringifier<Double>(conf, Double.class);
+ String sigma_jSigma_kString = stringifier.toString(sigma_jSigma_k);
log.info("Sigma_kSigma_j for each Label and for each Features");
- Float retSigma_jSigma_k = floatStringifier.fromString(sigma_jSigma_kString);
+ double retSigma_jSigma_k = stringifier.fromString(sigma_jSigma_kString);
log.info("{}", retSigma_jSigma_k);
conf.set("cnaivebayes.sigma_jSigma_k", sigma_jSigma_kString);
Path vocabCountFile = new Path(output+"/trainer-tfIdf/trainer-vocabCount/*");
- Float vocabCount = reader.readVocabCount(dfs, vocabCountFile, conf);
- String vocabCountString = floatStringifier.toString(vocabCount);
+ double vocabCount = reader.readVocabCount(dfs, vocabCountFile, conf);
+ String vocabCountString = stringifier.toString(vocabCount);
log.info("Vocabulary Count");
conf.set("cnaivebayes.vocabCount", vocabCountString);
- Float retvocabCount = floatStringifier.fromString(vocabCountString);
+ double retvocabCount = stringifier.fromString(vocabCountString);
log.info("{}", retvocabCount);
client.setConf(conf);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerMapper.java Fri Oct 24 04:47:42 2008
@@ -18,7 +18,7 @@
package org.apache.mahout.classifier.bayes;
import org.apache.hadoop.io.DefaultStringifier;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
@@ -34,21 +34,13 @@
import java.util.Map;
public class BayesThetaNormalizerMapper extends MapReduceBase implements
- Mapper<Text, FloatWritable, Text, FloatWritable> {
+ Mapper<Text, DoubleWritable, Text, DoubleWritable> {
private static final Logger log = LoggerFactory.getLogger(BayesThetaNormalizerMapper.class);
- private Map<String, Float> labelWeightSum = null;
-
- private String labelWeightSumString = " ";
-
- private Float sigma_jSigma_k = 0.0f;
-
- private String sigma_jSigma_kString = " ";
-
- private Float vocabCount = 0.0f;
-
- private String vocabCountString = " ";
+ private Map<String,Double> labelWeightSum = null;
+ private double sigma_jSigma_k = 0.0;
+ private double vocabCount = 0.0;
/**
* We need to calculate the thetaNormalization factor of each label
@@ -59,42 +51,42 @@
* @param reporter
* @throws IOException
*/
- public void map(Text key, FloatWritable value,
- OutputCollector<Text, FloatWritable> output, Reporter reporter)
+ public void map(Text key, DoubleWritable value,
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
- float alpha_i = 1.0f;
+ double alpha_i = 1.0;
String label = labelFeaturePair.split(",")[0];
- float weight = (float) Math.log((value.get() + alpha_i) / (labelWeightSum.get(label) + vocabCount));
- output.collect(new Text(("_" + label).trim()), new FloatWritable(weight));
+ double weight = Math.log((value.get() + alpha_i) / (labelWeightSum.get(label) + vocabCount));
+ output.collect(new Text(("_" + label).trim()), new DoubleWritable(weight));
}
@Override
public void configure(JobConf job) {
try {
if (labelWeightSum == null) {
- labelWeightSum = new HashMap<String, Float>();
+ labelWeightSum = new HashMap<String,Double>();
- DefaultStringifier<Map<String, Float>> mapStringifier = new DefaultStringifier<Map<String, Float>>(
+ DefaultStringifier<Map<String,Double>> mapStringifier = new DefaultStringifier<Map<String,Double>>(
job, GenericsUtil.getClass(labelWeightSum));
- labelWeightSumString = mapStringifier.toString(labelWeightSum);
+ String labelWeightSumString = mapStringifier.toString(labelWeightSum);
labelWeightSumString = job.get("cnaivebayes.sigma_k",
labelWeightSumString);
labelWeightSum = mapStringifier.fromString(labelWeightSumString);
- DefaultStringifier<Float> floatStringifier = new DefaultStringifier<Float>(
+ DefaultStringifier<Double> stringifier = new DefaultStringifier<Double>(
job, GenericsUtil.getClass(sigma_jSigma_k));
- sigma_jSigma_kString = floatStringifier.toString(sigma_jSigma_k);
+ String sigma_jSigma_kString = stringifier.toString(sigma_jSigma_k);
sigma_jSigma_kString = job.get("cnaivebayes.sigma_jSigma_k",
sigma_jSigma_kString);
- sigma_jSigma_k = floatStringifier.fromString(sigma_jSigma_kString);
+ sigma_jSigma_k = stringifier.fromString(sigma_jSigma_kString);
- vocabCountString = floatStringifier.toString(vocabCount);
+ String vocabCountString = stringifier.toString(vocabCount);
vocabCountString = job.get("cnaivebayes.vocabCount", vocabCountString);
- vocabCount = floatStringifier.fromString(vocabCountString);
+ vocabCount = stringifier.fromString(vocabCountString);
}
} catch (IOException ex) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/BayesThetaNormalizerReducer.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.classifier.bayes;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
@@ -26,41 +26,28 @@
import java.io.IOException;
import java.util.Iterator;
-import java.util.Map;
/**
* Can also be used as a local Combiner beacuse only two values should be there
* inside the values
*/
public class BayesThetaNormalizerReducer extends MapReduceBase implements
- Reducer<Text, FloatWritable, Text, FloatWritable> {
-
- private Map<String, Float> labelWeightSum = null;
-
- private String labelWeightSumString = " ";
-
- private Float sigma_jSigma_k = 0.0f;
-
- private String sigma_jSigma_kString = " ";
-
- private Float vocabCount = 0.0f;
-
- private String vocabCountString = " ";
+ Reducer<Text, DoubleWritable, Text, DoubleWritable> {
- public void reduce(Text key, Iterator<FloatWritable> values,
- OutputCollector<Text, FloatWritable> output, Reporter reporter)
+ public void reduce(Text key, Iterator<DoubleWritable> values,
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
// Key is label,word, value is the number of times we've seen this label
// word per local node. Output is the same
//String token = key.toString();
- float weightSumPerLabel = 0.0f;
+ double weightSumPerLabel = 0.0;
while (values.hasNext()) {
weightSumPerLabel += values.next().get();
}
- output.collect(key, new FloatWritable(weightSumPerLabel));
+ output.collect(key, new DoubleWritable(weightSumPerLabel));
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureDriver.java Fri Oct 24 04:47:42 2008
@@ -20,7 +20,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DefaultStringifier;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -65,7 +65,7 @@
JobConf conf = new JobConf(BayesFeatureDriver.class);
conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(FloatWritable.class);
+ conf.setOutputValueClass(DoubleWritable.class);
FileInputFormat.setInputPaths(conf, new Path(input));
Path outPath = new Path(output);
@@ -90,8 +90,7 @@
DefaultStringifier<Integer> intStringifier = new DefaultStringifier<Integer>(conf, Integer.class);
String gramSizeString = intStringifier.toString(gramSize);
- Integer retGramSize = intStringifier.fromString(gramSizeString);
- log.info("{}", retGramSize);
+ log.info("{}", intStringifier.fromString(gramSizeString));
conf.set("bayes.gramSize", gramSizeString);
client.setConf(conf);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureMapper.java Fri Oct 24 04:47:42 2008
@@ -18,7 +18,7 @@
package org.apache.mahout.classifier.bayes.common;
import org.apache.hadoop.io.DefaultStringifier;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
@@ -37,11 +37,11 @@
* Reads the input train set(preprocessed using the {@link BayesFileFormatter}).
*/
public class BayesFeatureMapper extends MapReduceBase implements
- Mapper<Text, Text, Text, FloatWritable> {
+ Mapper<Text, Text, Text, DoubleWritable> {
private static final Logger log = LoggerFactory.getLogger(BayesFeatureMapper.class);
- private static final FloatWritable one = new FloatWritable(1.0f);
+ private static final DoubleWritable one = new DoubleWritable(1.0);
private final Text labelWord = new Text();
@@ -62,7 +62,7 @@
* @throws IOException
*/
public void map(Text key, Text value,
- OutputCollector<Text, FloatWritable> output, Reporter reporter)
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
//String line = value.toString();
String label = key.toString();
@@ -91,8 +91,7 @@
String token = entry.getKey();
builder.append(',').append(token);
labelWord.set(builder.toString());
- FloatWritable f = new FloatWritable((float) (Math
- .log(1.0 + entry.getValue()) / lengthNormalisation));
+ DoubleWritable f = new DoubleWritable(Math.log(1.0 + entry.getValue()) / lengthNormalisation);
output.collect(labelWord, f);
builder.setLength(keyLen);// truncate back
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureOutputFormat.java Fri Oct 24 04:47:42 2008
@@ -32,17 +32,16 @@
/**
* This class extends the MultipleOutputFormat, allowing to write the output data to different output files in sequence file output format.
*/
-public class BayesFeatureOutputFormat extends
- MultipleOutputFormat<WritableComparable, Writable> {
+public class BayesFeatureOutputFormat extends MultipleOutputFormat<WritableComparable,Writable>{
- private SequenceFileOutputFormat theSequenceFileOutputFormat = null;
+ private SequenceFileOutputFormat<WritableComparable,Writable> theSequenceFileOutputFormat = null;
@Override
protected RecordWriter<WritableComparable, Writable> getBaseRecordWriter(
FileSystem fs, JobConf job, String name, Progressable arg3)
throws IOException {
if (theSequenceFileOutputFormat == null) {
- theSequenceFileOutputFormat = new SequenceFileOutputFormat();
+ theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable,Writable>();
}
return theSequenceFileOutputFormat.getRecordWriter(fs, job, name, arg3);
}
@@ -50,7 +49,7 @@
@Override
protected String generateFileNameForKeyValue(WritableComparable k, Writable v,
String name) {
- Text key = (Text)k;
+ Text key = (Text) k;
if(key.toString().startsWith("_"))
return "trainer-docCount/"+name;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesFeatureReducer.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.classifier.bayes.common;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
@@ -31,18 +31,18 @@
* Can also be used as a local Combiner. A simple summing reducer
*/
public class BayesFeatureReducer extends MapReduceBase
- implements Reducer<Text, FloatWritable, Text, FloatWritable> {
+ implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
public void reduce(Text key,
- Iterator<FloatWritable> values,
- OutputCollector<Text, FloatWritable> output,
+ Iterator<DoubleWritable> values,
+ OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the number of times we've seen this label word per local node. Output is the same
- float sum = 0;
+ double sum = 0.0;
while (values.hasNext()) {
sum += values.next().get();
}
- output.collect(key, new FloatWritable(sum));
+ output.collect(key, new DoubleWritable(sum));
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfDriver.java Fri Oct 24 04:47:42 2008
@@ -20,7 +20,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DefaultStringifier;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -45,7 +45,8 @@
* Takes in two arguments:
* <ol>
* <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the interim files as a {@link org.apache.hadoop.io.SequenceFile}</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the interim files as a
+ * {@link org.apache.hadoop.io.SequenceFile}</li>
* </ol>
* @param args The args
*/
@@ -68,7 +69,7 @@
conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(FloatWritable.class);
+ conf.setOutputValueClass(DoubleWritable.class);
SequenceFileInputFormat.addInputPath(conf, new Path(output + "/trainer-termDocCount"));
SequenceFileInputFormat.addInputPath(conf, new Path(output + "/trainer-wordFreq"));
@@ -94,14 +95,14 @@
Path interimFile = new Path(output+"/trainer-docCount/part-*");
- Map<String,Float> labelDocumentCounts= reader.readLabelDocumentCounts(dfs, interimFile, conf);
+ Map<String,Double> labelDocumentCounts= reader.readLabelDocumentCounts(dfs, interimFile, conf);
- DefaultStringifier<Map<String,Float>> mapStringifier =
- new DefaultStringifier<Map<String,Float>>(conf,GenericsUtil.getClass(labelDocumentCounts));
+ DefaultStringifier<Map<String,Double>> mapStringifier =
+ new DefaultStringifier<Map<String,Double>>(conf,GenericsUtil.getClass(labelDocumentCounts));
String labelDocumentCountString = mapStringifier.toString(labelDocumentCounts);
log.info("Counts of documents in Each Label");
- Map<String,Float> c = mapStringifier.fromString(labelDocumentCountString);
+ Map<String,Double> c = mapStringifier.fromString(labelDocumentCountString);
log.info("{}", c);
conf.set("cnaivebayes.labelDocumentCounts", labelDocumentCountString);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfMapper.java Fri Oct 24 04:47:42 2008
@@ -18,7 +18,7 @@
package org.apache.mahout.classifier.bayes.common;
import org.apache.hadoop.io.DefaultStringifier;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
@@ -34,12 +34,11 @@
import java.util.Map;
public class BayesTfIdfMapper extends MapReduceBase implements
- Mapper<Text, FloatWritable, Text, FloatWritable> {
+ Mapper<Text, DoubleWritable, Text, DoubleWritable> {
private static final Logger log = LoggerFactory.getLogger(BayesTfIdfMapper.class);
- private Map<String, Float> labelDocumentCounts = null;
- private String labelDocumentCountString = " ";
+ private Map<String,Double> labelDocumentCounts = null;
/**
* We need to calculate the Tf-Idf of each feature in each label
@@ -51,8 +50,8 @@
* @param reporter
* @throws IOException
*/
- public void map(Text key, FloatWritable value,
- OutputCollector<Text, FloatWritable> output, Reporter reporter)
+ public void map(Text key, DoubleWritable value,
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
@@ -61,17 +60,17 @@
labelFeaturePair = labelFeaturePair.substring(1);
String label = labelFeaturePair.split(",")[0];
- if(labelDocumentCounts.containsKey(label)==false){
+ if(labelDocumentCounts.containsKey(label) == false){
throw new IOException(label);
}
- float labelDocumentCount = labelDocumentCounts.get(label);
- float logIdf = (float) Math.log(labelDocumentCount / value.get());
+ double labelDocumentCount = labelDocumentCounts.get(label);
+ double logIdf = Math.log(labelDocumentCount / value.get());
- output.collect(new Text(labelFeaturePair), new FloatWritable(logIdf));
+ output.collect(new Text(labelFeaturePair), new DoubleWritable(logIdf));
} else if (labelFeaturePair.startsWith(",")) {
- output.collect(new Text("*vocabCount"), new FloatWritable(1.0f));
+ output.collect(new Text("*vocabCount"), new DoubleWritable(1.0));
} else {
output.collect(key, value);
}
@@ -81,12 +80,12 @@
public void configure(JobConf job) {
try {
if (labelDocumentCounts == null){
- labelDocumentCounts = new HashMap<String, Float>();
+ labelDocumentCounts = new HashMap<String,Double>();
- DefaultStringifier<Map<String,Float>> mapStringifier =
- new DefaultStringifier<Map<String,Float>>(job,GenericsUtil.getClass(labelDocumentCounts));
+ DefaultStringifier<Map<String,Double>> mapStringifier =
+ new DefaultStringifier<Map<String,Double>>(job,GenericsUtil.getClass(labelDocumentCounts));
- labelDocumentCountString = mapStringifier.toString(labelDocumentCounts);
+ String labelDocumentCountString = mapStringifier.toString(labelDocumentCounts);
labelDocumentCountString = job.get("cnaivebayes.labelDocumentCounts", labelDocumentCountString);
labelDocumentCounts = mapStringifier.fromString(labelDocumentCountString);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfOutputFormat.java Fri Oct 24 04:47:42 2008
@@ -34,24 +34,23 @@
* This class extends the MultipleOutputFormat, allowing to write the output data to
* different output files in sequence file output format.
*/
-public class BayesTfIdfOutputFormat extends
- MultipleOutputFormat<WritableComparable, Writable> {
+public class BayesTfIdfOutputFormat extends MultipleOutputFormat<WritableComparable,Writable> {
- private SequenceFileOutputFormat theSequenceFileOutputFormat = null;
+ private SequenceFileOutputFormat<WritableComparable,Writable> theSequenceFileOutputFormat = null;
@Override
protected RecordWriter<WritableComparable, Writable> getBaseRecordWriter(
FileSystem fs, JobConf job, String name, Progressable arg3)
throws IOException {
if (theSequenceFileOutputFormat == null) {
- theSequenceFileOutputFormat = new SequenceFileOutputFormat();
+ theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable,Writable>();
}
return theSequenceFileOutputFormat.getRecordWriter(fs, job, name, arg3);
}
@Override
protected String generateFileNameForKeyValue(WritableComparable k, Writable v,
String name) {
- Text key = (Text)k;
+ Text key = (Text) k;
if(key.toString().startsWith("*"))
return "trainer-vocabCount/"+name;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesTfIdfReducer.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.classifier.bayes.common;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
@@ -32,25 +32,25 @@
/**
* Can also be used as a local Combiner beacuse only two values should be there inside the values
*/
-public class BayesTfIdfReducer extends MapReduceBase implements Reducer<Text, FloatWritable, Text, FloatWritable> {
+public class BayesTfIdfReducer extends MapReduceBase implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
private static final Logger log = LoggerFactory.getLogger(BayesTfIdfReducer.class);
public void reduce(Text key,
- Iterator<FloatWritable> values,
- OutputCollector<Text, FloatWritable> output,
+ Iterator<DoubleWritable> values,
+ OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the number of times we've seen this label word per local node. Output is the same
String token = key.toString();
if(token.startsWith("*vocabCount")) {
- float vocabCount = 0;
+ double vocabCount = 0.0;
while (values.hasNext()) {
vocabCount += values.next().get();
}
log.info("{}\t{}", token, vocabCount);
- output.collect(key, new FloatWritable(vocabCount));
+ output.collect(key, new DoubleWritable(vocabCount));
} else {
- float idfTimes_D_ij = 1.0f;
+ double idfTimes_D_ij = 1.0;
//int numberofValues = 0;
while (values.hasNext()) {
idfTimes_D_ij *= values.next().get();
@@ -58,7 +58,7 @@
}
//if(numberofValues!=2) throw new IOException("Number of values should be exactly 2");
- output.collect(key, new FloatWritable(idfTimes_D_ij));
+ output.collect(key, new DoubleWritable(idfTimes_D_ij));
}
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerDriver.java Fri Oct 24 04:47:42 2008
@@ -19,7 +19,7 @@
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
@@ -37,7 +37,8 @@
* Takes in two arguments:
* <ol>
* <li>The input {@link org.apache.hadoop.fs.Path} where the input documents live</li>
- * <li>The output {@link org.apache.hadoop.fs.Path} where to write the the interim files as a {@link org.apache.hadoop.io.SequenceFile}</li>
+ * <li>The output {@link org.apache.hadoop.fs.Path} where to write the the interim files as a
+ * {@link org.apache.hadoop.io.SequenceFile}</li>
* </ol>
* @param args The args
*/
@@ -60,7 +61,7 @@
conf.setOutputKeyClass(Text.class);
- conf.setOutputValueClass(FloatWritable.class);
+ conf.setOutputValueClass(DoubleWritable.class);
SequenceFileInputFormat.addInputPath(conf, new Path(output + "/trainer-tfIdf/trainer-tfIdf"));
Path outPath = new Path(output + "/trainer-weights");
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerMapper.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.classifier.bayes.common;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
@@ -28,7 +28,7 @@
public class BayesWeightSummerMapper extends MapReduceBase implements
- Mapper<Text, FloatWritable, Text, FloatWritable> {
+ Mapper<Text, DoubleWritable, Text, DoubleWritable> {
/**
@@ -41,8 +41,8 @@
* @param reporter
* @throws IOException
*/
- public void map(Text key, FloatWritable value,
- OutputCollector<Text, FloatWritable> output, Reporter reporter)
+ public void map(Text key, DoubleWritable value,
+ OutputCollector<Text, DoubleWritable> output, Reporter reporter)
throws IOException {
String labelFeaturePair = key.toString();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerOutputFormat.java Fri Oct 24 04:47:42 2008
@@ -33,17 +33,16 @@
* This class extends the MultipleOutputFormat, allowing to write the output data to
* different output files in sequence file output format.
*/
-public class BayesWeightSummerOutputFormat extends
- MultipleOutputFormat<WritableComparable, Writable> {
+public class BayesWeightSummerOutputFormat extends MultipleOutputFormat<WritableComparable,Writable> {
- private SequenceFileOutputFormat theSequenceFileOutputFormat = null;
+ private SequenceFileOutputFormat<WritableComparable,Writable> theSequenceFileOutputFormat = null;
@Override
protected RecordWriter<WritableComparable, Writable> getBaseRecordWriter(
FileSystem fs, JobConf job, String name, Progressable arg3)
throws IOException {
if (theSequenceFileOutputFormat == null) {
- theSequenceFileOutputFormat = new SequenceFileOutputFormat();
+ theSequenceFileOutputFormat = new SequenceFileOutputFormat<WritableComparable,Writable>();
}
return theSequenceFileOutputFormat.getRecordWriter(fs, job, name, arg3);
}
@@ -51,7 +50,7 @@
@Override
protected String generateFileNameForKeyValue(WritableComparable k, Writable v,
String name) {
- Text key = (Text)k;
+ Text key = (Text) k;
if(key.toString().startsWith("*"))//sum of weight of all features for all label Sigma_kSigma_j
return "Sigma_kSigma_j/"+name;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/common/BayesWeightSummerReducer.java Fri Oct 24 04:47:42 2008
@@ -17,7 +17,7 @@
package org.apache.mahout.classifier.bayes.common;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.OutputCollector;
@@ -32,18 +32,18 @@
* Can also be used as a local Combiner
*/
public class BayesWeightSummerReducer extends MapReduceBase
- implements Reducer<Text, FloatWritable, Text, FloatWritable> {
+ implements Reducer<Text, DoubleWritable, Text, DoubleWritable> {
public void reduce(Text key,
- Iterator<FloatWritable> values,
- OutputCollector<Text, FloatWritable> output,
+ Iterator<DoubleWritable> values,
+ OutputCollector<Text, DoubleWritable> output,
Reporter reporter) throws IOException {
//Key is label,word, value is the tfidf of the feature of times we've seen this label word per local node. Output is the same
- float sum = 0;
+ double sum = 0.0;
while (values.hasNext()) {
sum += values.next().get();
}
- output.collect(key, new FloatWritable(sum));
+ output.collect(key, new DoubleWritable(sum));
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/bayes/io/SequenceFileModelReader.java Fri Oct 24 04:47:42 2008
@@ -21,7 +21,7 @@
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
@@ -62,7 +62,7 @@
public void loadWeightMatrix(Model model, FileSystem fs, Path pathPattern, Configuration conf) throws IOException {
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -87,7 +87,7 @@
Configuration conf) throws IOException {
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -111,7 +111,7 @@
public Model loadLabelWeights(Model model,FileSystem fs, Path pathPattern,
Configuration conf) throws IOException {
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -136,7 +136,7 @@
public Model loadThetaNormalizer(Model model,FileSystem fs, Path pathPattern,
Configuration conf) throws IOException {
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -161,7 +161,7 @@
Configuration conf) throws IOException {
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -187,9 +187,9 @@
throws IOException {
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
MapFile.Writer writer = new MapFile.Writer(conf, fs, "data.mapfile",
- Text.class, FloatWritable.class);
+ Text.class, DoubleWritable.class);
MapFile.Writer.setIndexInterval(conf, 3);
FileStatus[] outputFiles = fs.globStatus(pathPattern);
@@ -211,7 +211,7 @@
int idx = keyStr.indexOf(",");
if (idx != -1) {
// TODO srowen says data is not used?
- Map<String, Float> data = new HashMap<String, Float>();
+ Map<String,Double> data = new HashMap<String,Double>();
data.put(keyStr.substring(0, idx), value.get());
writer.append(new Text(key.toString()), value);
}
@@ -222,10 +222,10 @@
// return model;
}
- public Map<String, Float> readLabelSums(FileSystem fs, Path pathPattern, Configuration conf) throws IOException {
- Map<String, Float> labelSum = new HashMap<String, Float>();
+ public Map<String,Double> readLabelSums(FileSystem fs, Path pathPattern, Configuration conf) throws IOException {
+ Map<String,Double> labelSum = new HashMap<String,Double>();
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
@@ -245,11 +245,11 @@
return labelSum;
}
- public Map<String, Float> readLabelDocumentCounts(FileSystem fs, Path pathPattern, Configuration conf)
+ public Map<String,Double> readLabelDocumentCounts(FileSystem fs, Path pathPattern, Configuration conf)
throws IOException {
- Map<String, Float> labelDocumentCounts = new HashMap<String, Float>();
+ Map<String,Double> labelDocumentCounts = new HashMap<String,Double>();
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -268,11 +268,11 @@
return labelDocumentCounts;
}
- public Float readSigma_jSigma_k(FileSystem fs, Path pathPattern,
+ public double readSigma_jSigma_k(FileSystem fs, Path pathPattern,
Configuration conf) throws IOException {
- Map<String, Float> weightSum = new HashMap<String, Float>();
+ Map<String,Double> weightSum = new HashMap<String,Double>();
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
@@ -293,11 +293,11 @@
return weightSum.get("*");
}
- public Float readVocabCount(FileSystem fs, Path pathPattern,
+ public double readVocabCount(FileSystem fs, Path pathPattern,
Configuration conf) throws IOException {
- Map<String, Float> weightSum = new HashMap<String, Float>();
+ Map<String,Double> weightSum = new HashMap<String,Double>();
Writable key = new Text();
- FloatWritable value = new FloatWritable();
+ DoubleWritable value = new DoubleWritable();
FileStatus[] outputFiles = fs.globStatus(pathPattern);
for (FileStatus fileStatus : outputFiles) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java?rev=707606&r1=707605&r2=707606&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/classifier/cbayes/CBayesClassifier.java Fri Oct 24 04:47:42 2008
@@ -48,8 +48,8 @@
PriorityQueue pq = new ClassifierResultPriorityQueue(numResults);
ClassifierResult tmp;
for (String category : categories){
- float prob = documentProbability(model, category, document);
- if (prob < 0) {
+ double prob = documentProbability(model, category, document);
+ if (prob < 0.0) {
tmp = new ClassifierResult(category, prob);
pq.insert(tmp);
}
@@ -75,11 +75,11 @@
*/
public ClassifierResult classify(Model model, String[] document, String defaultCategory) {
ClassifierResult result = new ClassifierResult(defaultCategory);
- float min = 0.0f;
+ double min = 0.0;
Collection<String> categories = model.getLabels();
for (String category : categories) {
- float prob = documentProbability(model, category, document);
+ double prob = documentProbability(model, category, document);
if (prob < min) {
min = prob;
result.setLabel(category);
@@ -99,8 +99,8 @@
* @return The probability
* @see Model#FeatureWeight(String, String)
*/
- public float documentProbability(Model model, String label, String[] document) {
- float result = 0.0f;
+ public double documentProbability(Model model, String label, String[] document) {
+ double result = 0.0;
Map<String, Integer> wordList = new HashMap<String, Integer>(1000);
for (String word : document) {
if (wordList.containsKey(word)) {
@@ -129,8 +129,8 @@
ClassifierResult cr1 = (ClassifierResult) a;
ClassifierResult cr2 = (ClassifierResult) b;
- float score1 = cr1.getScore();
- float score2 = cr2.getScore();
+ double score1 = cr1.getScore();
+ double score2 = cr2.getScore();
return score1 == score2 ? cr1.getLabel().compareTo(cr2.getLabel()) < 0 : score1 < score2;
}
}