You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/12/07 21:47:29 UTC
svn commit: r888130 - in
/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop:
item/RecommenderJob.java pseudo/RecommenderJob.java
slopeone/SlopeOneAverageDiffsJob.java
Author: srowen
Date: Mon Dec 7 20:47:29 2009
New Revision: 888130
URL: http://svn.apache.org/viewvc?rev=888130&view=rev
Log:
Use gzip (not deflate?) for compressing text output
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java?rev=888130&r1=888129&r2=888130&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java Mon Dec 7 20:47:29 2009
@@ -20,6 +20,8 @@
import org.apache.commons.cli2.Option;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
@@ -124,6 +126,7 @@
recommenderConf.set(RecommenderMapper.COOCCURRENCE_PATH, cooccurrencePath);
recommenderConf.set(RecommenderMapper.ITEMID_INDEX_PATH, itemIDIndexPath);
recommenderConf.setInt(RecommenderMapper.RECOMMENDATIONS_PER_USER, recommendationsPerUser);
+ recommenderConf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);
JobClient.runJob(recommenderConf);
return 0;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java?rev=888130&r1=888129&r2=888130&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java Mon Dec 7 20:47:29 2009
@@ -20,6 +20,8 @@
import org.apache.commons.cli2.Option;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.TextInputFormat;
@@ -118,6 +120,7 @@
jobConf.set(RecommenderReducer.RECOMMENDER_CLASS_NAME, recommendClassName);
jobConf.setInt(RecommenderReducer.RECOMMENDATIONS_PER_USER, recommendationsPerUser);
jobConf.set(RecommenderReducer.DATA_MODEL_FILE, inputFile);
+ jobConf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);
JobClient.runJob(jobConf);
return 0;
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java?rev=888130&r1=888129&r2=888130&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java Mon Dec 7 20:47:29 2009
@@ -19,6 +19,8 @@
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
@@ -71,6 +73,7 @@
ItemItemWritable.class,
FloatWritable.class,
TextOutputFormat.class);
+ diffsToAveragesJobConf.setClass("mapred.output.compression.codec", GzipCodec.class, CompressionCodec.class);
JobClient.runJob(diffsToAveragesJobConf);
return 0;
}