You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/12/05 10:25:07 UTC

svn commit: r887521 - in /lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste: hadoop/AbstractJob.java hadoop/item/RecommenderJob.java impl/recommender/slopeone/file/FileDiffStorage.java

Author: srowen
Date: Sat Dec  5 09:25:05 2009
New Revision: 887521

URL: http://svn.apache.org/viewvc?rev=887521&view=rev
Log:
More progress on Hadoop job

Modified:
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
    lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java?rev=887521&r1=887520&r2=887521&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java Sat Dec  5 09:25:05 2009
@@ -17,7 +17,6 @@
 
 package org.apache.mahout.cf.taste.hadoop;
 
-import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli2.CommandLine;
 import org.apache.commons.cli2.Group;
 import org.apache.commons.cli2.Option;

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java?rev=887521&r1=887520&r2=887521&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java Sat Dec  5 09:25:05 2009
@@ -17,12 +17,72 @@
 
 package org.apache.mahout.cf.taste.hadoop.item;
 
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.SequenceFileInputFormat;
+import org.apache.hadoop.mapred.SequenceFileOutputFormat;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+import org.apache.hadoop.mapred.lib.IdentityReducer;
 import org.apache.mahout.cf.taste.hadoop.AbstractJob;
+import org.apache.mahout.cf.taste.hadoop.ItemPrefWritable;
+import org.apache.mahout.cf.taste.hadoop.RecommendedItemsWritable;
+import org.apache.mahout.cf.taste.hadoop.ToItemPrefsMapper;
+import org.apache.mahout.matrix.Vector;
+
+import java.util.Map;
 
 public final class RecommenderJob extends AbstractJob {
 
-  public static void main(String[] args) {
-    
+  public static void main(String[] args) throws Exception {
+
+    Map<String,Object> parsedArgs = parseArguments(args);
+
+    String prefsFile = parsedArgs.get("input").toString();
+    String outputPath = parsedArgs.get("output").toString();
+    String jarFile = parsedArgs.get("jarFile").toString();
+    String userVectorPath = outputPath + "/userVectors";
+    String cooccurrencePath = outputPath + "/cooccurrence";
+
+    JobConf toUserVectorConf = prepareJobConf(prefsFile,
+                                              userVectorPath,
+                                              jarFile,
+                                              TextInputFormat.class,
+                                              ToItemPrefsMapper.class,
+                                              LongWritable.class,
+                                              ItemPrefWritable.class,
+                                              ToUserVectorReducer.class,
+                                              LongWritable.class,
+                                              Vector.class,
+                                              SequenceFileOutputFormat.class);
+    JobClient.runJob(toUserVectorConf);
+
+    JobConf toCooccurrenceConf = prepareJobConf(userVectorPath,
+                                                cooccurrencePath,
+                                                jarFile,
+                                                SequenceFileInputFormat.class,
+                                                UserVectorToCooccurrenceMapper.class,
+                                                LongWritable.class,
+                                                LongWritable.class,
+                                                UserVectorToCooccurrenceReducer.class,
+                                                LongWritable.class,
+                                                Vector.class,
+                                                SequenceFileOutputFormat.class);
+    JobClient.runJob(toCooccurrenceConf);
+
+    JobConf recommenderConf = prepareJobConf(cooccurrencePath,
+                                             outputPath,
+                                             jarFile,
+                                             SequenceFileInputFormat.class,
+                                             RecommenderMapper.class,
+                                             LongWritable.class,
+                                             RecommendedItemsWritable.class,
+                                             IdentityReducer.class,
+                                             LongWritable.class,
+                                             RecommendedItemsWritable.class,
+                                             TextOutputFormat.class);
+    JobClient.runJob(recommenderConf);
   }
 
 }

Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java?rev=887521&r1=887520&r2=887521&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/slopeone/file/FileDiffStorage.java Sat Dec  5 09:25:05 2009
@@ -18,7 +18,6 @@
 package org.apache.mahout.cf.taste.impl.recommender.slopeone.file;
 
 import org.apache.mahout.cf.taste.common.Refreshable;
-import org.apache.mahout.cf.taste.hadoop.slopeone.SlopeOneDiffsToAveragesJob;
 import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
 import org.apache.mahout.cf.taste.impl.common.FastIDSet;
 import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
@@ -49,7 +48,7 @@
  *
  * <p>Commas or tabs can be delimiters. This is intended for use in conjuction
  * with the output of
- * {@link SlopeOneDiffsToAveragesJob}.</p>
+ * {@link org.apache.mahout.cf.taste.hadoop.slopeone.SlopeOneAverageDiffsJob}.</p>
  */
 public final class FileDiffStorage implements DiffStorage {