You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by ad...@apache.org on 2011/10/09 19:55:19 UTC
svn commit: r1180672 -
/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java
Author: adeneche
Date: Sun Oct 9 17:55:19 2011
New Revision: 1180672
URL: http://svn.apache.org/viewvc?rev=1180672&view=rev
Log:
MAHOUT-835 the Partial implementation avoids running unnecessary steps if the oob is not required
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java?rev=1180672&r1=1180671&r2=1180672&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/df/mapreduce/partial/PartialBuilder.java Sun Oct 9 17:55:19 2011
@@ -110,18 +110,24 @@ public class PartialBuilder extends Buil
Path outputPath = getOutputPath(conf);
- log.info("Computing partitions' first ids...");
- Step0Job step0 = new Step0Job(getOutputPath(conf), getDataPath(), getDatasetPath());
- Step0Output[] partitions = step0.run(new Configuration(conf));
-
- log.info("Processing the output...");
+ int[] firstIds = null;
TreeID[] keys = new TreeID[numTrees];
Node[] trees = new Node[numTrees];
- int[] firstIds = Step0Output.extractFirstIds(partitions);
- processOutput(job, outputPath, firstIds, keys, trees, callback);
+ Step0Output[] partitions = null;
+ int numMaps = 0;
+
+ if (callback != null) {
+ log.info("Computing partitions' first ids...");
+ Step0Job step0 = new Step0Job(getOutputPath(conf), getDataPath(), getDatasetPath());
+ partitions = step0.run(new Configuration(conf));
+
+ log.info("Processing the output...");
+ firstIds = Step0Output.extractFirstIds(partitions);
+
+ numMaps = partitions.length;
+ }
- // JobClient should have updated numMaps to the correct number of maps
- int numMaps = partitions.length;
+ processOutput(job, outputPath, firstIds, keys, trees, callback);
// call the second step in order to complete the oob predictions
if (callback != null && numMaps > 1 && isStep2(conf)) {
@@ -179,7 +185,9 @@ public class PartialBuilder extends Buil
if (trees != null) {
trees[index] = value.getTree();
}
- processOutput(firstIds, key, value, callback);
+ if (callback != null) {
+ processOutput(firstIds, key, value, callback);
+ }
index++;
}
}
@@ -201,12 +209,10 @@ public class PartialBuilder extends Buil
MapredOutput value,
PredictionCallback callback) {
- if (callback != null) {
- int[] predictions = value.getPredictions();
-
- for (int instanceId = 0; instanceId < predictions.length; instanceId++) {
- callback.prediction(key.treeId(), firstIds[key.partition()] + instanceId, predictions[instanceId]);
- }
+ int[] predictions = value.getPredictions();
+
+ for (int instanceId = 0; instanceId < predictions.length; instanceId++) {
+ callback.prediction(key.treeId(), firstIds[key.partition()] + instanceId, predictions[instanceId]);
}
}
}