You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/10/03 14:46:09 UTC
svn commit: r821305 [1/2] - in /lucene/mahout/trunk/core/src:
main/java/org/apache/mahout/common/ main/java/org/apache/mahout/df/
main/java/org/apache/mahout/df/builder/
main/java/org/apache/mahout/df/callback/
main/java/org/apache/mahout/df/data/ main...
Author: srowen
Date: Sat Oct 3 12:46:06 2009
New Revision: 821305
URL: http://svn.apache.org/viewvc?rev=821305&view=rev
Log:
MAHOUT-184
Added:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorException.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/RandomUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/Bagging.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DFUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DecisionForest.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/ErrorEstimate.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/DefaultTreeBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/TreeBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/ForestPredictions.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MeanTreeCollector.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MultiCallback.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/PredictionCallback.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/SingleTreePredictions.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Data.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataConverter.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataLoader.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Dataset.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorUtils.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Instance.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Equals.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/GreaterOrEquals.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Lesser.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/Builder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/MapredMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemInputFormat.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/PartialBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step0Job.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step1Mapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Job.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Mapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/Builder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredOutput.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemInputFormat.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemMapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/InterResults.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step0Job.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step1Mapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Job.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Mapper.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/CategoricalNode.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Leaf.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Node.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/NumericalNode.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/ref/SequentialBuilder.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/split/DefaultIgSplit.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/split/OptIgSplit.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/tools/Describe.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/builder/DefaultTreeBuilderTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/DataConverterTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/DataLoaderTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/DataTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/DatasetTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/DescriptorUtilsTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/data/Utils.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/inmem/InMemInputFormatTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/inmem/InMemInputSplitTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/PartialBuilderTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/PartialOutputCollector.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/PartialSequentialBuilder.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/PartitionBugTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/Step0JobTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/Step1MapperTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapred/partial/Step2MapperTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/inmem/InMemInputFormatTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/inmem/InMemInputSplitTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/InterResultsTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/MockContext.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/PartialBuilderTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/PartialSequentialBuilder.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/PartitionBugTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/Step0JobTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/Step1MapperTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/Step2MapperTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/mapreduce/partial/TreeIDTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/node/NodeTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/split/DefaultIgSplitTest.java
lucene/mahout/trunk/core/src/test/java/org/apache/mahout/df/split/OptIgSplitTest.java
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/RandomUtils.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/RandomUtils.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/RandomUtils.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/common/RandomUtils.java Sat Oct 3 12:46:06 2009
@@ -48,6 +48,20 @@
return testSeed ? new MersenneTwisterRNG(STANDARD_SEED) : new MersenneTwisterRNG();
}
+ public static Random getRandom(long seed) {
+ byte[] seedBytes = new byte[16];
+ seedBytes[0] = (byte) (seed >>> 56);
+ seedBytes[1] = (byte) (seed >>> 48);
+ seedBytes[2] = (byte) (seed >>> 40);
+ seedBytes[3] = (byte) (seed >>> 32);
+ seedBytes[4] = (byte) (seed >>> 24);
+ seedBytes[5] = (byte) (seed >>> 16);
+ seedBytes[6] = (byte) (seed >>> 8);
+ seedBytes[7] = (byte) seed;
+ System.arraycopy(seedBytes, 0, seedBytes, 8, 8);
+ return new MersenneTwisterRNG(seedBytes);
+ }
+
/** @return what {@link Double#hashCode()} would return for the same value */
public static int hashDouble(double value) {
// Just copied from Double.hashCode
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/Bagging.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/Bagging.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/Bagging.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/Bagging.java Sat Oct 3 12:46:06 2009
@@ -34,11 +34,11 @@
private static final Logger log = LoggerFactory.getLogger(Bagging.class);
- protected final TreeBuilder treeBuilder;
+ private final TreeBuilder treeBuilder;
- protected final Data data;
+ private final Data data;
- protected final boolean[] sampled;
+ private final boolean[] sampled;
public Bagging(TreeBuilder treeBuilder, Data data) {
this.treeBuilder = treeBuilder;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DFUtils.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DFUtils.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DFUtils.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DFUtils.java Sat Oct 3 12:46:06 2009
@@ -30,6 +30,8 @@
* Utility class that contains various helper methods
*/
public class DFUtils {
+ private DFUtils() {
+ }
/**
* Writes an Node[] into a DataOutput
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DecisionForest.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DecisionForest.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DecisionForest.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/DecisionForest.java Sat Oct 3 12:46:06 2009
@@ -32,7 +32,7 @@
*/
public class DecisionForest {
- protected final List<Node> trees;
+ private final List<Node> trees;
protected DecisionForest() {
trees = new ArrayList<Node>();
@@ -149,4 +149,9 @@
return trees.size() == rf.trees.size() && trees.containsAll(rf.trees);
}
+ @Override
+ public int hashCode() {
+ return trees.hashCode();
+ }
+
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/ErrorEstimate.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/ErrorEstimate.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/ErrorEstimate.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/ErrorEstimate.java Sat Oct 3 12:46:06 2009
@@ -21,6 +21,8 @@
* various methods to compute from the output of a random forest
*/
public class ErrorEstimate {
+ private ErrorEstimate() {
+ }
public static double errorRate(int[] labels, int[] predictions) {
assert labels.length == predictions.length : "labels.length != predictions.length";
@@ -49,9 +51,9 @@
public static int nbPredicted(int[] predictions) {
int nbpredicted = 0;
- for (int index = 0; index < predictions.length; index++) {
- if (predictions[index] != -1)
- nbpredicted++;
+ for (int prediction : predictions) {
+ if (prediction != -1)
+ nbpredicted++;
}
return nbpredicted;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/DefaultTreeBuilder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/DefaultTreeBuilder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/DefaultTreeBuilder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/DefaultTreeBuilder.java Sat Oct 3 12:46:06 2009
@@ -39,13 +39,13 @@
* <br>
* http://www.cs.cmu.edu/~awm/tutorials
*/
-public class DefaultTreeBuilder extends TreeBuilder {
+public class DefaultTreeBuilder implements TreeBuilder {
/** number of attributes to select randomly at each node */
- protected int m = 1;
+ private int m = 1;
/** IgSplit implementation */
- protected IgSplit igSplit;
+ private IgSplit igSplit;
public DefaultTreeBuilder() {
igSplit = new OptIgSplit();
@@ -114,11 +114,11 @@
assert m <= dataset.nbAttributes() : "m > num attributes";
int[] result = new int[m];
- int rvalue;
Arrays.fill(result, -1);
for (int index = 0; index < m; index++) {
+ int rvalue;
do {
rvalue = rng.nextInt(dataset.nbAttributes());
} while (ArrayUtils.contains(result, rvalue));
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/TreeBuilder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/TreeBuilder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/TreeBuilder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/builder/TreeBuilder.java Sat Oct 3 12:46:06 2009
@@ -25,7 +25,7 @@
/**
* Abstract base class for TreeBuilders
*/
-public abstract class TreeBuilder {
+public interface TreeBuilder {
/**
* Builds a Decision tree using the training data
@@ -34,6 +34,6 @@
* @param data training data
* @return root Node
*/
- public abstract Node build(Random rng, Data data);
+ Node build(Random rng, Data data);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/ForestPredictions.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/ForestPredictions.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/ForestPredictions.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/ForestPredictions.java Sat Oct 3 12:46:06 2009
@@ -25,10 +25,10 @@
/**
* Collects a forest's predictions
*/
-public class ForestPredictions extends PredictionCallback {
+public class ForestPredictions implements PredictionCallback {
/** predictions[n][label] = number of times instance n was classified 'label' */
- public final int[][] predictions;
+ private final int[][] predictions;
public ForestPredictions(int nbInstances, int nblabels) {
predictions = new int[nbInstances][];
@@ -65,6 +65,17 @@
return true;
}
+ @Override
+ public int hashCode() {
+ int hashCode = 1;
+ for (int[] row : predictions) {
+ for (int value : row) {
+ hashCode = 31 * hashCode + value;
+ }
+ }
+ return hashCode;
+ }
+
/**
* compute the prediction for each instance. the prediction of an instance is
* the index of the label that got most of the votes
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MeanTreeCollector.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MeanTreeCollector.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MeanTreeCollector.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MeanTreeCollector.java Sat Oct 3 12:46:06 2009
@@ -22,15 +22,15 @@
/**
* Computes the error rate for each tree, and returns the mean of all the trees
*/
-public class MeanTreeCollector extends PredictionCallback {
+public class MeanTreeCollector implements PredictionCallback {
/** number of errors for each tree */
- protected final int nbErrors[];
+ private final int[] nbErrors;
/** number of predictions for each tree */
- protected final int nbPredictions[];
+ private final int[] nbPredictions;
- protected final Data data;
+ private final Data data;
public MeanTreeCollector(Data data, int nbtrees) {
nbErrors = new int[nbtrees];
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MultiCallback.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MultiCallback.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MultiCallback.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/MultiCallback.java Sat Oct 3 12:46:06 2009
@@ -21,9 +21,9 @@
/**
* Combines many callbacks, that will be called when a prediction is done.
*/
-public class MultiCallback extends PredictionCallback {
+public class MultiCallback implements PredictionCallback {
- protected final PredictionCallback[] callbacks;
+ private final PredictionCallback[] callbacks;
public MultiCallback(PredictionCallback... callbacks) {
this.callbacks = callbacks;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/PredictionCallback.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/PredictionCallback.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/PredictionCallback.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/PredictionCallback.java Sat Oct 3 12:46:06 2009
@@ -21,7 +21,7 @@
/**
* Called each time an instance has been classified
*/
-public abstract class PredictionCallback {
+public interface PredictionCallback {
/**
* called when an instance has been classified
@@ -30,5 +30,5 @@
* @param instanceId classified instance
* @param prediction predicted label
*/
- public abstract void prediction(int treeId, int instanceId, int prediction);
+ void prediction(int treeId, int instanceId, int prediction);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/SingleTreePredictions.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/SingleTreePredictions.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/SingleTreePredictions.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/callback/SingleTreePredictions.java Sat Oct 3 12:46:06 2009
@@ -22,7 +22,7 @@
/**
* Collects the predictions for a single tree
*/
-public class SingleTreePredictions extends PredictionCallback {
+public class SingleTreePredictions implements PredictionCallback {
/** predictions[n] = 'label' predicted for instance 'n' */
public final int[] predictions;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Data.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Data.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Data.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Data.java Sat Oct 3 12:46:06 2009
@@ -37,7 +37,7 @@
*/
public class Data {
- protected final List<Instance> instances;
+ private final List<Instance> instances;
public final Dataset dataset;
@@ -158,10 +158,9 @@
public Data bagging(Random rng, boolean[] sampled) {
int datasize = size();
List<Instance> bag = new ArrayList<Instance>(datasize);
- int index;
for (int i = 0; i < datasize; i++) {
- index = rng.nextInt(datasize);
+ int index = rng.nextInt(datasize);
bag.add(instances.get(index));
sampled[index] = true;
}
@@ -174,7 +173,6 @@
* data. <b>VERY SLOW!</b>
*
* @param rng
- * @param ratio ratio of data to return
* @return
*/
public Data rsplit(Random rng, int subsize) {
@@ -251,6 +249,7 @@
return values;
}
+ @Override
public Data clone() {
return new Data(dataset, new ArrayList<Instance>(instances));
}
@@ -267,6 +266,11 @@
return instances.equals(data.instances) && dataset.equals(data.dataset);
}
+ @Override
+ public int hashCode() {
+ return instances.hashCode() + dataset.hashCode();
+ }
+
/**
* extract the labels of all instances
*
@@ -311,7 +315,6 @@
/**
* finds the majority label, breaking ties randomly
*
- * @param data
* @return the majority label value
*/
public int majorityLabel(Random rng) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataConverter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataConverter.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataConverter.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataConverter.java Sat Oct 3 12:46:06 2009
@@ -27,7 +27,7 @@
*/
public class DataConverter {
- protected final Dataset dataset;
+ private final Dataset dataset;
public DataConverter(Dataset dataset) {
this.dataset = dataset;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataLoader.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataLoader.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataLoader.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataLoader.java Sat Oct 3 12:46:06 2009
@@ -46,7 +46,10 @@
public class DataLoader {
private static final Logger log = LoggerFactory.getLogger(DataLoader.class);
-
+
+ private DataLoader() {
+ }
+
/**
* Converts a comma-separated String to a Vector.
*
@@ -120,11 +123,10 @@
* @param fs file system
* @param fpath data file path
* @return
- * @throws Exception if any problem is encountered
+ * @throws IOException if any problem is encountered
*/
- @SuppressWarnings("unchecked")
- public static Data loadData(Dataset dataset, FileSystem fs, Path fpath) throws Exception {
+ public static Data loadData(Dataset dataset, FileSystem fs, Path fpath) throws IOException {
FSDataInputStream input = fs.open(fpath);
Scanner scanner = new Scanner(input);
@@ -156,14 +158,8 @@
/**
* Loads the data from a String array
- *
- * @param dataset
- * @param data
- * @return
- * @throws Exception if any problem is encountered
*/
- @SuppressWarnings("unchecked")
- public static Data loadData(Dataset dataset, String[] data) throws Exception {
+ public static Data loadData(Dataset dataset, String[] data) {
List<Instance> instances = new ArrayList<Instance>();
DataConverter converter = new DataConverter(dataset);
@@ -192,12 +188,9 @@
* @param descriptor attributes description
* @param fs file system
* @param path data path
- * @return
- * @throws IOException
*/
- @SuppressWarnings("unchecked")
public static Dataset generateDataset(String descriptor, FileSystem fs, Path path)
- throws Exception {
+ throws DescriptorException, IOException {
Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor);
FSDataInputStream input = fs.open(path);
@@ -227,23 +220,20 @@
* Generates the Dataset by parsing the entire data
* @param descriptor attributes description
* @param data
- * @return
- * @throws Exception
*/
- @SuppressWarnings("unchecked")
- public static Dataset generateDataset(String descriptor, String[] data) throws Exception {
+ public static Dataset generateDataset(String descriptor, String[] data) throws DescriptorException {
Attribute[] attrs = DescriptorUtils.parseDescriptor(descriptor);
// used to convert CATEGORICAL and LABEL attributes to Integer
List<String>[] values = new List[attrs.length];
int id = 0;
- for (int index = 0; index < data.length; index++) {
- if (data[index].isEmpty()) {
+ for (String aData : data) {
+ if (aData.isEmpty()) {
continue;
}
-
- if (parseString(id, attrs, values, data[index]) != null) {
+
+ if (parseString(id, attrs, values, aData) != null) {
id++;
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataUtils.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataUtils.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataUtils.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DataUtils.java Sat Oct 3 12:46:06 2009
@@ -25,6 +25,8 @@
* Helper methods that deals with data lists and arrays of values
*/
public class DataUtils {
+ private DataUtils() {
+ }
/**
* Computes the sum of the values
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Dataset.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Dataset.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Dataset.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Dataset.java Sat Oct 3 12:46:06 2009
@@ -125,7 +125,7 @@
return ignored;
}
- protected Dataset() {
+ private Dataset() {
}
/**
@@ -144,10 +144,10 @@
attributes = new Attribute[nbattrs];
this.values = new String[nbattrs][];
ignored = new int[attrs.length - (nbattrs + 1)]; // nbignored = total - (nbattrs + label)
-
- int ind = 0;
+
labelId = -1;
int ignoredId = 0;
+ int ind = 0;
for (int attr = 0; attr < attrs.length; attr++) {
if (attrs[attr].isIgnored()) {
ignored[ignoredId++] = attr;
@@ -188,16 +188,16 @@
*/
protected static int countAttributes(Attribute[] attrs) {
int nbattrs = 0;
-
- for (int attr = 0; attr < attrs.length; attr++) {
- if (attrs[attr].isNumerical() || attrs[attr].isCategorical())
+
+ for (Attribute attr1 : attrs) {
+ if (attr1.isNumerical() || attr1.isCategorical())
nbattrs++;
}
return nbattrs;
}
- protected static void validateValues(Attribute[] attrs, List<String>[] values) {
+ private static void validateValues(Attribute[] attrs, List<String>[] values) {
assert attrs.length == values.length : "attrs.length != values.length";
for (int attr = 0; attr < attrs.length; attr++) {
@@ -252,11 +252,24 @@
return labelId == dataset.labelId && nbInstances == dataset.nbInstances;
}
+ public int hashCode() {
+ int hashCode = labelId + 31 * nbInstances;
+ for (Attribute attr : attributes) {
+ hashCode = 31 * hashCode + attr.hashCode();
+ }
+ for (String label : labels) {
+ hashCode = 31 * hashCode + label.hashCode();
+ }
+ for (String[] valueRow : values) {
+ for (String value : valueRow) {
+ hashCode = 31 * hashCode + value.hashCode();
+ }
+ }
+ return hashCode;
+ }
+
/**
* Loads the dataset from a file
- * @param fs
- * @param path
- * @return
* @throws IOException
*/
public static Dataset load(Configuration conf, Path path) throws IOException {
@@ -264,7 +277,7 @@
FSDataInputStream input = fs.open(path);
- Dataset dataset = Dataset.read(input);
+ Dataset dataset = read(input);
input.close();
return dataset;
@@ -277,6 +290,7 @@
return dataset;
}
+ @Override
public void readFields(DataInput in) throws IOException {
int nbAttributes = in.readInt();
attributes = new Attribute[nbAttributes];
@@ -301,6 +315,7 @@
nbInstances = in.readInt();
}
+ @Override
public void write(DataOutput out) throws IOException {
out.writeInt(attributes.length); // nb attributes
for (Attribute attr:attributes) {
Added: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorException.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorException.java?rev=821305&view=auto
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorException.java (added)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorException.java Sat Oct 3 12:46:06 2009
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.df.data;
+
+/**
+ * Exception thrown when parsing a descriptor
+ */
+public class DescriptorException extends Exception {
+ public DescriptorException(String msg) {
+ super(msg);
+ }
+}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorUtils.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorUtils.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorUtils.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/DescriptorUtils.java Sat Oct 3 12:46:06 2009
@@ -13,18 +13,9 @@
* Contains various methods that deal with descriptor strings
*/
public class DescriptorUtils {
-
- /**
- * Exception throwed when parsing a descriptor
- *
- */
- @SuppressWarnings("serial")
- public static class DescriptorException extends Exception {
- public DescriptorException(String msg) {
- super(msg);
- }
+ private DescriptorUtils() {
}
-
+
/**
* Parses a descriptor string and generates the corresponding array of Attributes
*
@@ -79,16 +70,15 @@
* @throws DescriptorException
*/
public static String generateDescriptor(List<String> tokens) throws DescriptorException {
- StringBuffer descriptor = new StringBuffer();
+ StringBuilder descriptor = new StringBuilder();
int multiplicator = 0;
- int number;
-
+
for (String token : tokens) {
try {
// try to parse an integer
- number = Integer.parseInt(token);
-
+ int number = Integer.parseInt(token);
+
if (number <= 0) {
throw new DescriptorException("Multiplicator ("+number+") must be > 0");
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Instance.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Instance.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Instance.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/Instance.java Sat Oct 3 12:46:06 2009
@@ -28,7 +28,7 @@
public final int id;
/** attributes, except LABEL and IGNORED */
- protected final Vector attrs;
+ private final Vector attrs;
/**
* instance label code.<br>
@@ -75,6 +75,11 @@
return id == instance.id && label == instance.label && attrs.equals(instance.attrs);
}
+
+ @Override
+ public int hashCode() {
+ return id + label + attrs.hashCode();
+ }
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Equals.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Equals.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Equals.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Equals.java Sat Oct 3 12:46:06 2009
@@ -24,9 +24,9 @@
*/
public class Equals extends Condition {
- protected final int attr;
+ private final int attr;
- protected final double value;
+ private final double value;
public Equals(int attr, double value) {
this.attr = attr;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/GreaterOrEquals.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/GreaterOrEquals.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/GreaterOrEquals.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/GreaterOrEquals.java Sat Oct 3 12:46:06 2009
@@ -24,9 +24,9 @@
*/
public class GreaterOrEquals extends Condition {
- protected final int attr;
+ private final int attr;
- protected final double value;
+ private final double value;
public GreaterOrEquals(int attr, double value) {
this.attr = attr;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Lesser.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Lesser.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Lesser.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/data/conditions/Lesser.java Sat Oct 3 12:46:06 2009
@@ -24,9 +24,9 @@
*/
public class Lesser extends Condition {
- protected final int attr;
+ private final int attr;
- protected final double value;
+ private final double value;
public Lesser(int attr, double value) {
this.attr = attr;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/Builder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/Builder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/Builder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/Builder.java Sat Oct 3 12:46:06 2009
@@ -167,7 +167,7 @@
return new Path(fs.getWorkingDirectory(), outputDirName);
}
- public Builder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath,
+ protected Builder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath,
Long seed, Configuration conf) {
this.treeBuilder = treeBuilder;
this.dataPath = dataPath;
@@ -186,8 +186,7 @@
*/
public static Path getDistributedCacheFile(JobConf job, int index)
throws IOException {
- URI[] files = new URI[0];
- files = DistributedCache.getCacheFiles(job);
+ URI[] files = DistributedCache.getCacheFiles(job);
if (files == null || files.length < index) {
throw new IOException("path not found in the DistributedCache");
@@ -223,11 +222,8 @@
/**
* Sequential implementation should override this method to simulate the job
* execution
- *
- * @param job
- * @throws Exception
*/
- protected void runJob(JobConf job) throws Exception {
+ protected void runJob(JobConf job) throws IOException {
JobClient.runJob(job);
}
@@ -243,8 +239,7 @@
protected abstract DecisionForest parseOutput(JobConf job,
PredictionCallback callback) throws IOException;
- public DecisionForest build(int nbTrees, PredictionCallback callback)
- throws Exception {
+ public DecisionForest build(int nbTrees, PredictionCallback callback) throws IOException {
JobConf job = new JobConf(conf, Builder.class);
Path outputPath = getOutputPath(job);
@@ -290,6 +285,7 @@
*/
public static void sortSplits(InputSplit[] splits) {
Arrays.sort(splits, new Comparator<InputSplit>() {
+ @Override
public int compare(InputSplit a, InputSplit b) {
try {
long left = a.getLength();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/MapredMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/MapredMapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/MapredMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/MapredMapper.java Sat Oct 3 12:46:06 2009
@@ -31,7 +31,7 @@
*/
public class MapredMapper extends MapReduceBase {
- protected final static Logger log = LoggerFactory.getLogger(MapredMapper.class);
+ protected static final Logger log = LoggerFactory.getLogger(MapredMapper.class);
private boolean noOutput;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemBuilder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemBuilder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemBuilder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemBuilder.java Sat Oct 3 12:46:06 2009
@@ -110,19 +110,19 @@
* @param callback
* @return
*/
- protected DecisionForest processOutput(Map<Integer, MapredOutput> output,
+ private static DecisionForest processOutput(Map<Integer, MapredOutput> output,
PredictionCallback callback) {
List<Node> trees = new ArrayList<Node>();
- for (Integer key : output.keySet()) {
- MapredOutput value = output.get(key);
+ for (Map.Entry<Integer, MapredOutput> entry : output.entrySet()) {
+ MapredOutput value = entry.getValue();
trees.add(value.getTree());
if (callback != null) {
int[] predictions = value.getPredictions();
for (int index = 0; index < predictions.length; index++) {
- callback.prediction(key, index, predictions[index]);
+ callback.prediction(entry.getKey(), index, predictions[index]);
}
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemInputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemInputFormat.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemInputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemInputFormat.java Sat Oct 3 12:46:06 2009
@@ -30,6 +30,7 @@
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RecordReader;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.df.mapred.Builder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,13 +43,13 @@
*/
public class InMemInputFormat implements InputFormat<IntWritable, NullWritable> {
- protected static final Logger log = LoggerFactory.getLogger(InMemInputSplit.class);
+ private static final Logger log = LoggerFactory.getLogger(InMemInputSplit.class);
- protected Random rng;
+ private Random rng;
- protected Long seed;
+ private Long seed;
- protected boolean isSingleSeed;
+ private boolean isSingleSeed;
/**
* Used for DEBUG purposes only. if true and a seed is available, all the
@@ -58,7 +59,7 @@
* @param conf
* @return
*/
- public static boolean isSingleSeed(Configuration conf) {
+ private static boolean isSingleSeed(Configuration conf) {
return conf.getBoolean("debug.mahout.rf.single.seed", false);
}
@@ -81,7 +82,7 @@
+ "this can lead to no-repeatable behavior");
}
- rng = (seed == null || isSingleSeed) ? null : new Random(seed);
+ rng = (seed == null || isSingleSeed) ? null : RandomUtils.getRandom();
int id = 0;
@@ -145,7 +146,7 @@
@Override
public float getProgress() throws IOException {
if (pos == 0)
- return 0f;
+ return 0.0f;
else
return (float) (pos - 1) / split.nbTrees;
}
@@ -168,6 +169,8 @@
*/
public static class InMemInputSplit implements InputSplit {
+ private static final String[] NO_LOCATIONS = new String[0];
+
/** Id of the first tree of this split */
private int firstId;
@@ -218,7 +221,7 @@
@Override
public String[] getLocations() throws IOException {
- return new String[0];
+ return NO_LOCATIONS;
}
@Override
@@ -238,6 +241,11 @@
}
@Override
+ public int hashCode() {
+ return firstId + nbTrees + (seed == null ? 0 : seed.intValue());
+ }
+
+ @Override
public String toString() {
return String.format("[firstId:%d, nbTrees:%d, seed:%d]", firstId,
nbTrees, seed);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemMapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/inmem/InMemMapper.java Sat Oct 3 12:46:06 2009
@@ -29,6 +29,7 @@
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.StringUtils;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.df.Bagging;
import org.apache.mahout.df.callback.SingleTreePredictions;
import org.apache.mahout.df.data.Data;
@@ -48,11 +49,11 @@
public class InMemMapper extends MapredMapper implements
Mapper<IntWritable, NullWritable, IntWritable, MapredOutput> {
- protected Bagging bagging;
+ private Bagging bagging;
- protected Random rng;
+ private Random rng;
- protected Data data;
+ private Data data;
/**
* Load the training data
@@ -61,7 +62,7 @@
* @return
* @throws RuntimeException if the data could not be loaded
*/
- protected Data loadData(JobConf conf, Dataset dataset) {
+ private static Data loadData(JobConf conf, Dataset dataset) {
try {
Path dataPath = Builder.getDistributedCacheFile(conf, 1);
FileSystem fs = FileSystem.get(dataPath.toUri(), conf);
@@ -122,9 +123,9 @@
log.debug("Initialising rng with seed : " + seed);
if (seed == null)
- rng = new Random();
+ rng = RandomUtils.getRandom();
else
- rng = new Random(seed);
+ rng = RandomUtils.getRandom(seed);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/PartialBuilder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/PartialBuilder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/PartialBuilder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/PartialBuilder.java Sat Oct 3 12:46:06 2009
@@ -47,7 +47,7 @@
*/
public class PartialBuilder extends Builder {
- private final static Logger log = LoggerFactory.getLogger(PartialBuilder.class);
+ private static final Logger log = LoggerFactory.getLogger(PartialBuilder.class);
/**
* Indicates if we should run the second step of the builder.<br>
@@ -143,7 +143,6 @@
* @param outputPath directory that contains the output of the job
* @param firstIds partitions' first ids in hadoop's order
* @param keys
- * @param values
* @param callback can be null
* @throws IOException
*/
@@ -192,8 +191,6 @@
* callback
*
* @param firstIds partitions' first ids in hadoop's order
- * @param keys
- * @param values
* @param callback
* @return
*/
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step0Job.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step0Job.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step0Job.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step0Job.java Sat Oct 3 12:46:06 2009
@@ -17,8 +17,6 @@
package org.apache.mahout.df.mapred.partial;
-import static org.apache.mahout.df.DFUtils.listOutputFiles;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -45,6 +43,7 @@
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.mapred.SequenceFileOutputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.mahout.df.DFUtils;
/**
* preparation step of the partial mapreduce builder. Computes some stats that
@@ -53,16 +52,15 @@
public class Step0Job {
/** directory that will hold this job's output */
- protected final Path outputPath;
+ private final Path outputPath;
/** file that contains the serialized dataset */
- protected final Path datasetPath;
+ private final Path datasetPath;
/** directory that contains the data used in the first step */
- protected final Path dataPath;
+ private final Path dataPath;
/**
- * @param conf
* @param base base directory
* @param dataPath data used in the first step
* @param datasetPath
@@ -90,7 +88,7 @@
// put the dataset into the DistributedCache
// use setCacheFiles() to overwrite the first-step cache files
- URI[] files = new URI[] { datasetPath.toUri() };
+ URI[] files = { datasetPath.toUri() };
DistributedCache.setCacheFiles(files, job);
FileInputFormat.setInputPaths(job, dataPath);
@@ -123,7 +121,7 @@
int numMaps = job.getNumMapTasks();
FileSystem fs = outputPath.getFileSystem(job);
- Path[] outfiles = listOutputFiles(fs, outputPath);
+ Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath);
int[] keys = new int[numMaps];
Step0Output[] values = new Step0Output[numMaps];
@@ -219,6 +217,7 @@
}
}
+ @Override
public void map(LongWritable key, Text value,
OutputCollector<IntWritable, Step0Output> output, Reporter reporter)
throws IOException {
@@ -257,25 +256,42 @@
/** number of instances in the partition */
protected int size;
- public Step0Output(long firstId, int size) {
+ protected Step0Output(long firstId, int size) {
this.firstId = firstId;
this.size = size;
}
+ @Override
public void readFields(DataInput in) throws IOException {
firstId = in.readLong();
size = in.readInt();
}
+ @Override
public void write(DataOutput out) throws IOException {
out.writeLong(firstId);
out.writeInt(size);
}
+ @Override
protected Step0Output clone() {
return new Step0Output(firstId, size);
}
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Step0Output)) {
+ return false;
+ }
+ return firstId == ((Step0Output) other).firstId;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) firstId;
+ }
+
+ @Override
public int compareTo(Step0Output obj) {
if (firstId < obj.firstId)
return -1;
@@ -299,7 +315,7 @@
int[] sizes = new int[partitions.length];
for (int p = 0; p < partitions.length; p++) {
- sizes[p] = (int) partitions[p].size;
+ sizes[p] = partitions[p].size;
}
return sizes;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step1Mapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step1Mapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step1Mapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step1Mapper.java Sat Oct 3 12:46:06 2009
@@ -28,6 +28,7 @@
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.df.Bagging;
import org.apache.mahout.df.callback.SingleTreePredictions;
import org.apache.mahout.df.data.Data;
@@ -52,24 +53,24 @@
private static final Logger log = LoggerFactory.getLogger(Step1Mapper.class);
/** used to convert input values to data instances */
- protected DataConverter converter;
+ private DataConverter converter;
- protected Random rng;
+ private Random rng;
/** number of trees to be built by this mapper */
- protected int nbTrees;
+ private int nbTrees;
/** id of the first tree */
protected int firstTreeId;
/** mapper's partition */
- protected int partition;
+ private int partition;
/** used by close() */
- protected OutputCollector<TreeID, MapredOutput> output;
+ private OutputCollector<TreeID, MapredOutput> output;
/** will contain all instances if this mapper's split */
- protected final List<Instance> instances = new ArrayList<Instance>();
+ private final List<Instance> instances = new ArrayList<Instance>();
@Override
public void configure(JobConf job) {
@@ -94,9 +95,9 @@
// prepare random-numders generator
log.debug("seed : " + seed);
if (seed == null)
- rng = new Random();
- else
- rng = new Random(seed);
+ rng = RandomUtils.getRandom();
+ else
+ rng = RandomUtils.getRandom(seed);
// mapper's partition
if (partition < 0) {
@@ -128,14 +129,15 @@
* @return
*/
public static int nbTrees(int numMaps, int numTrees, int partition) {
- int nbTrees = (int) Math.floor(numTrees / numMaps);
+ int nbTrees = numTrees / numMaps;
if (partition == 0) {
- nbTrees += (int) numTrees - nbTrees * numMaps;
+ nbTrees += numTrees - nbTrees * numMaps;
}
return nbTrees;
}
+ @Override
public void map(LongWritable key, Text value,
OutputCollector<TreeID, MapredOutput> output, Reporter reporter)
throws IOException {
@@ -154,11 +156,11 @@
Data data = new Data(getDataset(), instances);
Bagging bagging = new Bagging(getTreeBuilder(), data);
- SingleTreePredictions callback = null;
- int[] predictions = null;
TreeID key = new TreeID();
log.debug("Building " + nbTrees + " trees");
+ SingleTreePredictions callback = null;
+ int[] predictions = null;
for (int treeId = 0; treeId < nbTrees; treeId++) {
log.debug("Building tree N° : " + treeId);
if (isOobEstimate() && !isNoOutput()) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Job.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Job.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Job.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Job.java Sat Oct 3 12:46:06 2009
@@ -44,22 +44,21 @@
public class Step2Job {
/** directory that will hold this job's output */
- protected final Path outputPath;
+ private final Path outputPath;
/** file that will contains the forest, passed to the maps */
- protected final Path forestPath;
+ private final Path forestPath;
/** file that contains the serialized dataset */
- protected final Path datasetPath;
+ private final Path datasetPath;
/** directory that contains the data used in the first step */
- protected final Path dataPath;
+ private final Path dataPath;
/** partitions info in Hadoop's order */
- protected final Step0Output[] partitions;
+ private final Step0Output[] partitions;
/**
- * @param conf
* @param base base directory
* @param dataPath data used in the first step
* @param datasetPath
@@ -106,7 +105,7 @@
// put the dataset and the forest into the DistributedCache
// use setCacheFiles() to overwrite the first-step cache files
- URI[] files = new URI[] { datasetPath.toUri(), forestPath.toUri() };
+ URI[] files = { datasetPath.toUri(), forestPath.toUri() };
DistributedCache.setCacheFiles(files, job);
FileInputFormat.setInputPaths(job, dataPath);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Mapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Mapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Mapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapred/partial/Step2Mapper.java Sat Oct 3 12:46:06 2009
@@ -52,26 +52,26 @@
private static final Logger log = LoggerFactory.getLogger(Step2Mapper.class);
- protected TreeID[] keys;
+ private TreeID[] keys;
- protected Node[] trees;
+ private Node[] trees;
- protected SingleTreePredictions[] callbacks;
+ private SingleTreePredictions[] callbacks;
- protected DataConverter converter;
+ private DataConverter converter;
- protected int partition = -1;
+ private int partition = -1;
/** used by close() */
- protected OutputCollector<TreeID, MapredOutput> output;
+ private OutputCollector<TreeID, MapredOutput> output;
/** num treated instances */
- protected int instanceId;
+ private int instanceId;
@Override
public void configure(JobConf job) {
// get the cached files' paths
- URI[] files = new URI[0];
+ URI[] files;
try {
files = DistributedCache.getCacheFiles(job);
} catch (IOException e) {
@@ -175,6 +175,7 @@
}
+ @Override
public void map(LongWritable key, Text value,
OutputCollector<TreeID, MapredOutput> output, Reporter reporter)
throws IOException {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/Builder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/Builder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/Builder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/Builder.java Sat Oct 3 12:46:06 2009
@@ -64,7 +64,7 @@
private final Configuration conf;
- protected String outputDirName = "output";
+ private String outputDirName = "output";
protected int numTrees;
@@ -103,7 +103,7 @@
return conf.getBoolean("mahout.rf.oob", false);
}
- protected static void setOobEstimate(Configuration conf, boolean value) {
+ private static void setOobEstimate(Configuration conf, boolean value) {
conf.setBoolean("mahout.rf.oob", value);
}
@@ -127,7 +127,7 @@
* @param conf configuration
* @param seed random seed
*/
- protected static void setRandomSeed(Configuration conf, long seed) {
+ private static void setRandomSeed(Configuration conf, long seed) {
conf.setLong("mahout.rf.random.seed", seed);
}
@@ -139,7 +139,7 @@
return (TreeBuilder) StringUtils.fromString(string);
}
- protected static void setTreeBuilder(Configuration conf,
+ private static void setTreeBuilder(Configuration conf,
TreeBuilder treeBuilder) {
conf.set("mahout.rf.treebuilder", StringUtils.toString(treeBuilder));
}
@@ -223,7 +223,7 @@
return Dataset.load(conf, datasetPath);
}
- public Builder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath,
+ protected Builder(TreeBuilder treeBuilder, Path dataPath, Path datasetPath,
Long seed, Configuration conf) {
this.treeBuilder = treeBuilder;
this.dataPath = dataPath;
@@ -249,9 +249,8 @@
*
* @param job Hadoop's job
* @return true is the job succeeded
- * @throws Exception if the job encounters an error
*/
- protected boolean runJob(Job job) throws Exception {
+ protected boolean runJob(Job job) throws ClassNotFoundException, IOException, InterruptedException {
return job.waitForCompletion(true);
}
@@ -262,12 +261,12 @@
* @param job Hadoop's job
* @param callback can be null
* @return Built DecisionForest
- * @throws Exception if anything goes wrong while parsing the output
+ * @throws IOException if anything goes wrong while parsing the output
*/
- protected abstract DecisionForest parseOutput(Job job, PredictionCallback callback) throws Exception;
+ protected abstract DecisionForest parseOutput(Job job, PredictionCallback callback) throws IOException, ClassNotFoundException, InterruptedException;
public DecisionForest build(int nbTrees, PredictionCallback callback)
- throws Exception {
+ throws IOException, ClassNotFoundException, InterruptedException {
numTrees = getNbTrees(conf);
Path outputPath = getOutputPath(conf);
@@ -318,6 +317,7 @@
*/
public static void sortSplits(InputSplit[] splits) {
Arrays.sort(splits, new Comparator<InputSplit>() {
+ @Override
public int compare(InputSplit a, InputSplit b) {
try {
long left = a.getLength();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredMapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredMapper.java Sat Oct 3 12:46:06 2009
@@ -32,7 +32,7 @@
public class MapredMapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> extends
Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT> {
- protected final static Logger log = LoggerFactory.getLogger(MapredMapper.class);
+ protected static final Logger log = LoggerFactory.getLogger(MapredMapper.class);
private boolean noOutput;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredOutput.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredOutput.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredOutput.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/MapredOutput.java Sat Oct 3 12:46:06 2009
@@ -32,9 +32,9 @@
*/
public class MapredOutput implements Writable {
- protected Node tree;
+ private Node tree;
- protected int[] predictions;
+ private int[] predictions;
public Node getTree() {
return tree;
@@ -113,8 +113,17 @@
}
@Override
+ public int hashCode() {
+ int hashCode = tree == null ? 1 : tree.hashCode();
+ for (int prediction : predictions) {
+ hashCode = 31 * hashCode + prediction;
+ }
+ return hashCode;
+ }
+
+ @Override
public String toString() {
- return "{" + tree + " | " + Arrays.toString(predictions) + "}";
+ return "{" + tree + " | " + Arrays.toString(predictions) + '}';
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemBuilder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemBuilder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemBuilder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemBuilder.java Sat Oct 3 12:46:06 2009
@@ -117,19 +117,19 @@
* @param callback
* @return
*/
- protected DecisionForest processOutput(Map<Integer, MapredOutput> output,
+ private static DecisionForest processOutput(Map<Integer, MapredOutput> output,
PredictionCallback callback) {
List<Node> trees = new ArrayList<Node>();
- for (Integer key : output.keySet()) {
- MapredOutput value = output.get(key);
+ for (Map.Entry<Integer, MapredOutput> entry : output.entrySet()) {
+ MapredOutput value = entry.getValue();
trees.add(value.getTree());
if (callback != null) {
int[] predictions = value.getPredictions();
for (int index = 0; index < predictions.length; index++) {
- callback.prediction(key, index, predictions[index]);
+ callback.prediction(entry.getKey(), index, predictions[index]);
}
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemInputFormat.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemInputFormat.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemInputFormat.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemInputFormat.java Sat Oct 3 12:46:06 2009
@@ -33,6 +33,7 @@
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.df.mapreduce.Builder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,14 +46,14 @@
*/
public class InMemInputFormat extends InputFormat<IntWritable, NullWritable> {
- protected static final Logger log = LoggerFactory
+ private static final Logger log = LoggerFactory
.getLogger(InMemInputSplit.class);
- protected Random rng;
+ private Random rng;
- protected Long seed;
+ private Long seed;
- protected boolean isSingleSeed;
+ private boolean isSingleSeed;
/**
* Used for DEBUG purposes only. if true and a seed is available, all the
@@ -62,7 +63,7 @@
* @param conf
* @return
*/
- public static boolean isSingleSeed(Configuration conf) {
+ private static boolean isSingleSeed(Configuration conf) {
return conf.getBoolean("debug.mahout.rf.single.seed", false);
}
@@ -82,8 +83,7 @@
return getSplits(conf, numSplits);
}
- public List<InputSplit> getSplits(Configuration conf, int numSplits) throws IOException,
- InterruptedException {
+ public List<InputSplit> getSplits(Configuration conf, int numSplits) {
int nbTrees = Builder.getNbTrees(conf);
int splitSize = nbTrees / numSplits;
@@ -95,7 +95,7 @@
+ "this can lead to no-repeatable behavior");
}
- rng = (seed == null || isSingleSeed) ? null : new Random(seed);
+ rng = (seed == null || isSingleSeed) ? null : RandomUtils.getRandom(seed);
int id = 0;
@@ -143,7 +143,7 @@
@Override
public float getProgress() throws IOException {
if (pos == 0)
- return 0f;
+ return 0.0f;
else
return (float) (pos - 1) / split.nbTrees;
}
@@ -186,6 +186,8 @@
*/
public static class InMemInputSplit extends InputSplit implements Writable {
+ private static final String[] NO_LOCATIONS = new String[0];
+
/** Id of the first tree of this split */
private int firstId;
@@ -236,7 +238,7 @@
@Override
public String[] getLocations() throws IOException {
- return new String[0];
+ return NO_LOCATIONS;
}
@Override
@@ -256,6 +258,11 @@
}
@Override
+ public int hashCode() {
+ return firstId + nbTrees + (seed == null ? 0 : seed.intValue());
+ }
+
+ @Override
public String toString() {
return String.format("[firstId:%d, nbTrees:%d, seed:%d]", firstId,
nbTrees, seed);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemMapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemMapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemMapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/inmem/InMemMapper.java Sat Oct 3 12:46:06 2009
@@ -26,6 +26,7 @@
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.util.StringUtils;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.df.Bagging;
import org.apache.mahout.df.callback.SingleTreePredictions;
import org.apache.mahout.df.data.Data;
@@ -45,11 +46,11 @@
public class InMemMapper extends
MapredMapper<IntWritable, NullWritable, IntWritable, MapredOutput> {
- protected Bagging bagging;
+ private Bagging bagging;
- protected Random rng;
+ private Random rng;
- protected Data data;
+ private Data data;
/**
* Load the training data
@@ -58,7 +59,7 @@
* @return
* @throws RuntimeException if the data could not be loaded
*/
- protected Data loadData(Configuration conf, Dataset dataset) {
+ private static Data loadData(Configuration conf, Dataset dataset) {
try {
Path dataPath = Builder.getDistributedCacheFile(conf, 1);
FileSystem fs = FileSystem.get(dataPath.toUri(), conf);
@@ -119,9 +120,9 @@
log.debug("Initialising rng with seed : " + seed);
if (seed == null)
- rng = new Random();
+ rng = RandomUtils.getRandom();
else
- rng = new Random(seed);
+ rng = RandomUtils.getRandom(seed);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/InterResults.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/InterResults.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/InterResults.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/InterResults.java Sat Oct 3 12:46:06 2009
@@ -30,6 +30,8 @@
* methods are protected.<br>
*/
public class InterResults {
+ private InterResults() {
+ }
/**
* Load the trees and the keys returned from the first step
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java Sat Oct 3 12:46:06 2009
@@ -46,7 +46,7 @@
*/
public class PartialBuilder extends Builder {
- private final static Logger log = LoggerFactory.getLogger(PartialBuilder.class);
+ private static final Logger log = LoggerFactory.getLogger(PartialBuilder.class);
/**
* Indicates if we should run the second step of the builder.<br>
@@ -103,7 +103,7 @@
@Override
protected DecisionForest parseOutput(Job job, PredictionCallback callback)
- throws Exception {
+ throws IOException, ClassNotFoundException, InterruptedException {
Configuration conf = job.getConfiguration();
int numTrees = getNbTrees(conf);
@@ -143,7 +143,6 @@
* @param outputPath directory that contains the output of the job
* @param firstIds partitions' first ids in hadoop's order
* @param keys
- * @param values
* @param callback can be null
* @throws IOException
*/
@@ -194,8 +193,6 @@
* callback
*
* @param firstIds partitions' first ids in hadoop's order
- * @param keys
- * @param values
* @param callback
* @return
*/
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step0Job.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step0Job.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step0Job.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step0Job.java Sat Oct 3 12:46:06 2009
@@ -17,8 +17,6 @@
package org.apache.mahout.df.mapreduce.partial;
-import static org.apache.mahout.df.DFUtils.listOutputFiles;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
@@ -43,6 +41,7 @@
import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat;
+import org.apache.mahout.df.DFUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,7 +63,6 @@
protected final Path dataPath;
/**
- * @param conf
* @param base base directory
* @param dataPath data used in the first step
* @param datasetPath
@@ -80,9 +78,8 @@
*
* @param conf configuration
* @return partitions' info in Hadoop's order
- * @throws Exception
*/
- public Step0Output[] run(Configuration conf) throws Exception {
+ public Step0Output[] run(Configuration conf) throws IOException, ClassNotFoundException, InterruptedException {
// check the output
if (outputPath.getFileSystem(conf).exists(outputPath))
@@ -90,7 +87,7 @@
// put the dataset into the DistributedCache
// use setCacheFiles() to overwrite the first-step cache files
- URI[] files = new URI[] { datasetPath.toUri() };
+ URI[] files = { datasetPath.toUri() };
DistributedCache.setCacheFiles(files, conf);
Job job = new Job(conf);
@@ -116,9 +113,7 @@
/**
* Extracts the output and processes it
- *
- * @param conf
- *
+ *
* @return info for each partition in Hadoop's order
* @throws IOException
*/
@@ -129,7 +124,7 @@
FileSystem fs = outputPath.getFileSystem(conf);
- Path[] outfiles = listOutputFiles(fs, outputPath);
+ Path[] outfiles = DFUtils.listOutputFiles(fs, outputPath);
List<Integer> keys = new ArrayList<Integer>();
List<Step0Output> values = new ArrayList<Step0Output>();
@@ -253,25 +248,42 @@
/** number of instances in the partition */
protected int size;
- public Step0Output(long firstId, int size) {
+ protected Step0Output(long firstId, int size) {
this.firstId = firstId;
this.size = size;
}
+ @Override
public void readFields(DataInput in) throws IOException {
firstId = in.readLong();
size = in.readInt();
}
+ @Override
public void write(DataOutput out) throws IOException {
out.writeLong(firstId);
out.writeInt(size);
}
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof Step0Output)) {
+ return false;
+ }
+ return firstId == ((Step0Output) other).firstId;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) firstId;
+ }
+
+ @Override
protected Step0Output clone() {
return new Step0Output(firstId, size);
}
+ @Override
public int compareTo(Step0Output obj) {
if (firstId < obj.firstId)
return -1;
@@ -295,7 +307,7 @@
int[] sizes = new int[partitions.length];
for (int p = 0; p < partitions.length; p++) {
- sizes[p] = (int) partitions[p].size;
+ sizes[p] = partitions[p].size;
}
return sizes;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step1Mapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step1Mapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step1Mapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step1Mapper.java Sat Oct 3 12:46:06 2009
@@ -25,6 +25,7 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
+import org.apache.mahout.common.RandomUtils;
import org.apache.mahout.df.Bagging;
import org.apache.mahout.df.callback.SingleTreePredictions;
import org.apache.mahout.df.data.Data;
@@ -48,21 +49,21 @@
private static final Logger log = LoggerFactory.getLogger(Step1Mapper.class);
/** used to convert input values to data instances */
- protected DataConverter converter;
+ private DataConverter converter;
- protected Random rng;
+ private Random rng;
/** number of trees to be built by this mapper */
- protected int nbTrees;
+ private int nbTrees;
/** id of the first tree */
protected int firstTreeId;
/** mapper's partition */
- protected int partition;
+ private int partition;
/** will contain all instances if this mapper's split */
- protected final List<Instance> instances = new ArrayList<Instance>();
+ private final List<Instance> instances = new ArrayList<Instance>();
@Override
protected void setup(Context context) throws IOException,
@@ -89,9 +90,9 @@
// prepare random-numders generator
log.debug("seed : " + seed);
if (seed == null)
- rng = new Random();
+ rng = RandomUtils.getRandom();
else
- rng = new Random(seed);
+ rng = RandomUtils.getRandom(seed);
// mapper's partition
if (partition < 0) {
@@ -123,9 +124,9 @@
* @return
*/
public static int nbTrees(int numMaps, int numTrees, int partition) {
- int nbTrees = (int) Math.floor(numTrees / numMaps);
+ int nbTrees = numTrees / numMaps;
if (partition == 0) {
- nbTrees += (int) numTrees - nbTrees * numMaps;
+ nbTrees += numTrees - nbTrees * numMaps;
}
return nbTrees;
@@ -146,11 +147,11 @@
Data data = new Data(getDataset(), instances);
Bagging bagging = new Bagging(getTreeBuilder(), data);
- SingleTreePredictions callback = null;
- int[] predictions = null;
TreeID key = new TreeID();
log.debug("Building " + nbTrees + " trees");
+ SingleTreePredictions callback = null;
+ int[] predictions = null;
for (int treeId = 0; treeId < nbTrees; treeId++) {
log.debug("Building tree N° : " + treeId);
if (isOobEstimate() && !isNoOutput()) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Job.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Job.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Job.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Job.java Sat Oct 3 12:46:06 2009
@@ -41,22 +41,21 @@
public class Step2Job {
/** directory that will hold this job's output */
- protected final Path outputPath;
+ private final Path outputPath;
/** file that will contains the forest, passed to the maps */
- protected final Path forestPath;
+ private final Path forestPath;
/** file that contains the serialized dataset */
- protected final Path datasetPath;
+ private final Path datasetPath;
/** directory that contains the data used in the first step */
- protected final Path dataPath;
+ private final Path dataPath;
/** partitions info in Hadoop's order */
- protected final Step0Output[] partitions;
+ private final Step0Output[] partitions;
/**
- * @param conf
* @param base base directory
* @param dataPath data used in the first step
* @param datasetPath
@@ -77,10 +76,9 @@
* @param keys keys returned by the first step
* @param trees trees returned by the first step
* @param callback
- * @throws Exception
*/
- public void run(Configuration conf, TreeID[] keys, Node[] trees, PredictionCallback callback)
- throws Exception {
+ public void run(Configuration conf, TreeID[] keys, Node[] trees, PredictionCallback callback)
+ throws IOException, ClassNotFoundException, InterruptedException {
if (callback == null) {
// no need to launch the job
return;
@@ -101,7 +99,7 @@
// put the dataset and the forest into the DistributedCache
// use setCacheFiles() to overwrite the first-step cache files
- URI[] files = new URI[] { datasetPath.toUri(), forestPath.toUri() };
+ URI[] files = { datasetPath.toUri(), forestPath.toUri() };
DistributedCache.setCacheFiles(files, conf);
Job job = new Job(conf);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Mapper.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Mapper.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Mapper.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/Step2Mapper.java Sat Oct 3 12:46:06 2009
@@ -47,18 +47,18 @@
private static final Logger log = LoggerFactory.getLogger(Step2Mapper.class);
- protected TreeID[] keys;
+ private TreeID[] keys;
- protected Node[] trees;
+ private Node[] trees;
- protected SingleTreePredictions[] callbacks;
+ private SingleTreePredictions[] callbacks;
- protected DataConverter converter;
+ private DataConverter converter;
- protected int partition = -1;
+ private int partition = -1;
/** num treated instances */
- protected int instanceId;
+ private int instanceId;
@Override
@@ -66,7 +66,7 @@
Configuration conf = context.getConfiguration();
// get the cached files' paths
- URI[] files = new URI[0];
+ URI[] files;
try {
files = DistributedCache.getCacheFiles(conf);
} catch (IOException e) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/CategoricalNode.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/CategoricalNode.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/CategoricalNode.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/CategoricalNode.java Sat Oct 3 12:46:06 2009
@@ -17,13 +17,12 @@
package org.apache.mahout.df.node;
-import static org.apache.commons.lang.ArrayUtils.indexOf;
-
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.Arrays;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.mahout.df.DFUtils;
import org.apache.mahout.df.data.Instance;
@@ -45,7 +44,7 @@
@Override
public int classify(Instance instance) {
- int index = indexOf(values, instance.get(attr));
+ int index = ArrayUtils.indexOf(values, instance.get(attr));
if (index == -1) {
// value not available, we cannot predict
return -1;
@@ -90,10 +89,21 @@
&& Arrays.equals(childs, node.childs);
}
+ @Override
+ public int hashCode() {
+ int hashCode = attr;
+ for (double value : values) {
+ hashCode = 31 * hashCode + (int) Double.doubleToLongBits(value);
+ }
+ for (Node node : childs) {
+ hashCode = 31 * hashCode + node.hashCode();
+ }
+ return hashCode;
+ }
@Override
protected String getString() {
- StringBuffer buffer = new StringBuffer();
+ StringBuilder buffer = new StringBuilder();
for (Node child:childs) {
buffer.append(child).append(',');
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Leaf.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Leaf.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Leaf.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Leaf.java Sat Oct 3 12:46:06 2009
@@ -76,6 +76,11 @@
return label == leaf.label;
}
+
+ @Override
+ public int hashCode() {
+ return label;
+ }
@Override
protected String getString() {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Node.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Node.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Node.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/Node.java Sat Oct 3 12:46:06 2009
@@ -29,11 +29,11 @@
*/
public abstract class Node implements Writable {
- protected final static String DELIMITER = ",";
+ //protected static final String DELIMITER = ",";
- protected static enum NODE_TYPE {
+ protected enum NODE_TYPE {
MOCKLEAF, LEAF, NUMERICAL, CATEGORICAL
- };
+ }
/**
* predicts the label for the instance
@@ -77,7 +77,7 @@
}
}
- public static final Node read(DataInput in) throws IOException {
+ public static Node read(DataInput in) throws IOException {
NODE_TYPE type = NODE_TYPE.values()[in.readInt()];
Node node;
@@ -106,7 +106,7 @@
@Override
public final String toString() {
- return node2Type() + ":" + getString() + ";";
+ return node2Type() + ":" + getString() + ';';
}
protected abstract String getString();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/NumericalNode.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/NumericalNode.java?rev=821305&r1=821304&r2=821305&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/NumericalNode.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/df/node/NumericalNode.java Sat Oct 3 12:46:06 2009
@@ -80,10 +80,14 @@
&& loChild.equals(node.loChild) && hiChild.equals(node.hiChild);
}
+ @Override
+ public int hashCode() {
+ return attr + (int) Double.doubleToLongBits(split) + loChild.hashCode() + hiChild.hashCode();
+ }
@Override
protected String getString() {
- return loChild.toString() + "," + hiChild.toString();
+ return loChild.toString() + ',' + hiChild.toString();
}
@Override