You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/12/05 10:25:07 UTC
svn commit: r887521 - in
/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste:
hadoop/AbstractJob.java hadoop/item/RecommenderJob.java
impl/recommender/slopeone/file/FileDiffStorage.java
Author: srowen
Date: Sat Dec 5 09:25:05 2009
New Revision: 887521
URL: http://svn.apache.org/viewvc?rev=887521&view=rev
Log:
More progress on Hadoop job
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java?rev=887521&r1=887520&r2=887521&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java Sat Dec 5 09:25:05 2009
@@ -17,7 +17,6 @@
package org.apache.mahout.cf.taste.hadoop;
-import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java?rev=887521&r1=887520&r2=887521&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java Sat Dec 5 09:25:05 2009
@@ -17,12 +17,72 @@
package org.apache.mahout.cf.taste.hadoop.item;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.mahout.cf.taste.hadoop.AbstractJob;
+import org.apache.mahout.cf.taste.hadoop.ItemPrefWritable;
+import org.apache.mahout.cf.taste.hadoop.RecommendedItemsWritable;
+import org.apache.mahout.cf.taste.hadoop.ToItemPrefsMapper;
+import org.apache.mahout.matrix.Vector;
+
+import java.util.Map;
public final class RecommenderJob extends AbstractJob {
- public static void main(String[] args) {
-
+ public static void main(String[] args) throws Exception {
+
+ Map<String,Object> parsedArgs = parseArguments(args);
+
+ String prefsFile = parsedArgs.get("input").toString();
+ String outputPath = parsedArgs.get("output").toString();
+ String jarFile = parsedArgs.get("jarFile").toString();
+ String userVectorPath = outputPath + "/userVectors";
+ String cooccurrencePath = outputPath + "/cooccurrence";
+
+ JobConf toUserVectorConf = prepareJobConf(prefsFile,
+ userVectorPath,
+ jarFile,
+ TextInputFormat.class,
+ ToItemPrefsMapper.class,
+ LongWritable.class,
+ ItemPrefWritable.class,
+ ToUserVectorReducer.class,
+ LongWritable.class,
+ Vector.class,
+ SequenceFileOutputFormat.class);
+ JobClient.runJob(toUserVectorConf);
+
+ JobConf toCooccurrenceConf = prepareJobConf(userVectorPath,
+ cooccurrencePath,
+ jarFile,
+ SequenceFileInputFormat.class,
+ UserVectorToCooccurrenceMapper.class,
+ LongWritable.class,
+ LongWritable.class,
+ UserVectorToCooccurrenceReducer.class,
+ LongWritable.class,
+ Vector.class,
+ SequenceFileOutputFormat.class);
+ JobClient.runJob(toCooccurrenceConf);
+
+ JobConf recommenderConf = prepareJobConf(cooccurrencePath,
+ outputPath,
+ jarFile,
+ SequenceFileInputFormat.class,
+ RecommenderMapper.class,
+ LongWritable.class,
+ RecommendedItemsWritable.class,
+ IdentityReducer.class,
+ LongWritable.class,
+ RecommendedItemsWritable.class,
+ TextOutputFormat.class);
+ JobClient.runJob(recommenderConf);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java?rev=887521&r1=887520&r2=887521&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java Sat Dec 5 09:25:05 2009
@@ -18,7 +18,6 @@
package org.apache.mahout.cf.taste.impl.recommender.slopeone.file;
import org.apache.mahout.cf.taste.common.Refreshable;
-import org.apache.mahout.cf.taste.hadoop.slopeone.SlopeOneDiffsToAveragesJob;
import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
@@ -49,7 +48,7 @@
*
* <p>Commas or tabs can be delimiters. This is intended for use in conjuction
* with the output of
- * {@link SlopeOneDiffsToAveragesJob}.</p>
+ * {@link org.apache.mahout.cf.taste.hadoop.slopeone.SlopeOneAverageDiffsJob}.</p>
*/
public final class FileDiffStorage implements DiffStorage {