You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2010/03/31 13:47:56 UTC
svn commit: r929484 - in /lucene/mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/hadoop/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/
core/src/main/java/org/apache/m...
Author: srowen
Date: Wed Mar 31 11:47:55 2010
New Revision: 929484
URL: http://svn.apache.org/viewvc?rev=929484&view=rev
Log:
Property use Configured / getConf() in AbstractJob. Remove jarFile option as it seems to be guessable from proper use of JobConf constructor, from Mapper class
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java
lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/text/TextParagraphSplittingJob.java
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java?rev=929484&r1=929483&r2=929484&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/AbstractJob.java Wed Mar 31 11:47:55 2010
@@ -30,7 +30,7 @@ import org.apache.commons.cli2.builder.A
import org.apache.commons.cli2.builder.DefaultOptionBuilder;
import org.apache.commons.cli2.builder.GroupBuilder;
import org.apache.commons.cli2.commandline.Parser;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Writable;
@@ -46,22 +46,10 @@ import org.apache.mahout.common.commandl
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public abstract class AbstractJob implements Tool {
+public abstract class AbstractJob extends Configured implements Tool {
private static final Logger log = LoggerFactory.getLogger(AbstractJob.class);
-
- private Configuration configuration;
-
- @Override
- public Configuration getConf() {
- return configuration;
- }
-
- @Override
- public void setConf(Configuration configuration) {
- this.configuration = configuration;
- }
-
+
protected static Option buildOption(String name, String shortName, String description) {
return buildOption(name, shortName, description, true, null);
}
@@ -90,10 +78,9 @@ public abstract class AbstractJob implem
Option tempDirOpt = buildOption("tempDir", "t", "Intermediate output directory", "temp");
Option outputOpt = DefaultOptionCreator.outputOption().create();
Option helpOpt = DefaultOptionCreator.helpOption();
- Option jarFileOpt = buildOption("jarFile", "m", "Implementation jar", false, null);
-
+
GroupBuilder gBuilder = new GroupBuilder().withName("Options").withOption(inputOpt)
- .withOption(tempDirOpt).withOption(outputOpt).withOption(helpOpt).withOption(jarFileOpt);
+ .withOption(tempDirOpt).withOption(outputOpt).withOption(helpOpt);
for (Option opt : extraOpts) {
gBuilder = gBuilder.withOption(opt);
@@ -118,7 +105,7 @@ public abstract class AbstractJob implem
}
Map<String,String> result = new HashMap<String,String>();
- maybePut(result, cmdLine, inputOpt, tempDirOpt, outputOpt, helpOpt, jarFileOpt);
+ maybePut(result, cmdLine, inputOpt, tempDirOpt, outputOpt, helpOpt);
maybePut(result, cmdLine, extraOpts);
return result;
@@ -133,28 +120,22 @@ public abstract class AbstractJob implem
}
}
- protected static JobConf prepareJobConf(String inputPath,
- String outputPath,
- String jarFile,
- Class<? extends InputFormat> inputFormat,
- Class<? extends Mapper> mapper,
- Class<? extends Writable> mapperKey,
- Class<? extends Writable> mapperValue,
- Class<? extends Reducer> reducer,
- Class<? extends Writable> reducerKey,
- Class<? extends Writable> reducerValue,
- Class<? extends OutputFormat> outputFormat) throws IOException {
+ protected JobConf prepareJobConf(String inputPath,
+ String outputPath,
+ Class<? extends InputFormat> inputFormat,
+ Class<? extends Mapper> mapper,
+ Class<? extends Writable> mapperKey,
+ Class<? extends Writable> mapperValue,
+ Class<? extends Reducer> reducer,
+ Class<? extends Writable> reducerKey,
+ Class<? extends Writable> reducerValue,
+ Class<? extends OutputFormat> outputFormat) throws IOException {
- JobConf jobConf = new JobConf();
+ JobConf jobConf = new JobConf(getConf(), mapper);
FileSystem fs = FileSystem.get(jobConf);
Path inputPathPath = new Path(inputPath).makeQualified(fs);
Path outputPathPath = new Path(outputPath).makeQualified(fs);
-
- if(jarFile != null) {
- jobConf.set("mapred.jar", jarFile);
- jobConf.setJar(jarFile);
- }
jobConf.setClass("mapred.input.format.class", inputFormat, InputFormat.class);
jobConf.set("mapred.input.dir", StringUtils.escapeString(inputPathPath.toString()));
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java?rev=929484&r1=929483&r2=929484&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java Wed Mar 31 11:47:55 2010
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Map;
import org.apache.commons.cli2.Option;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.compress.CompressionCodec;
@@ -76,7 +77,6 @@ public final class RecommenderJob extend
String inputPath = parsedArgs.get("--input");
String tempDirPath = parsedArgs.get("--tempDir");
String outputPath = parsedArgs.get("--output");
- String jarFile = parsedArgs.get("--jarFile");
int recommendationsPerUser = Integer.parseInt(parsedArgs.get("--numRecommendations"));
String usersFile = parsedArgs.get("--usersFile");
@@ -84,23 +84,23 @@ public final class RecommenderJob extend
String itemIDIndexPath = tempDirPath + "/itemIDIndex";
String cooccurrencePath = tempDirPath + "/cooccurrence";
- JobConf itemIDIndexConf = AbstractJob.prepareJobConf(inputPath, itemIDIndexPath, jarFile,
+ JobConf itemIDIndexConf = prepareJobConf(inputPath, itemIDIndexPath,
TextInputFormat.class, ItemIDIndexMapper.class, IntWritable.class, LongWritable.class,
ItemIDIndexReducer.class, IntWritable.class, LongWritable.class, MapFileOutputFormat.class);
JobClient.runJob(itemIDIndexConf);
- JobConf toUserVectorConf = AbstractJob.prepareJobConf(inputPath, userVectorPath, jarFile,
+ JobConf toUserVectorConf = prepareJobConf(inputPath, userVectorPath,
TextInputFormat.class, ToItemPrefsMapper.class, LongWritable.class, ItemPrefWritable.class,
ToUserVectorReducer.class, LongWritable.class, VectorWritable.class, SequenceFileOutputFormat.class);
JobClient.runJob(toUserVectorConf);
- JobConf toCooccurrenceConf = AbstractJob.prepareJobConf(userVectorPath, cooccurrencePath, jarFile,
+ JobConf toCooccurrenceConf = prepareJobConf(userVectorPath, cooccurrencePath,
SequenceFileInputFormat.class, UserVectorToCooccurrenceMapper.class, IntWritable.class,
IntWritable.class, UserVectorToCooccurrenceReducer.class, IntWritable.class, VectorWritable.class,
MapFileOutputFormat.class);
JobClient.runJob(toCooccurrenceConf);
- JobConf recommenderConf = AbstractJob.prepareJobConf(userVectorPath, outputPath, jarFile,
+ JobConf recommenderConf = prepareJobConf(userVectorPath, outputPath,
SequenceFileInputFormat.class, RecommenderMapper.class, LongWritable.class,
RecommendedItemsWritable.class, IdentityReducer.class, LongWritable.class,
RecommendedItemsWritable.class, TextOutputFormat.class);
@@ -116,7 +116,7 @@ public final class RecommenderJob extend
}
public static void main(String[] args) throws Exception {
- ToolRunner.run(new RecommenderJob(), args);
+ ToolRunner.run(new Configuration(), new RecommenderJob(), args);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java?rev=929484&r1=929483&r2=929484&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java Wed Mar 31 11:47:55 2010
@@ -114,7 +114,6 @@ public final class RecommenderJob extend
}
String inputFile = parsedArgs.get("--input");
String outputPath = parsedArgs.get("--output");
- String jarFile = parsedArgs.get("--jarFile");
String usersFile = parsedArgs.get("--usersFile");
if (usersFile == null) {
usersFile = inputFile;
@@ -123,7 +122,7 @@ public final class RecommenderJob extend
String recommendClassName = parsedArgs.get("--recommenderClassName");
int recommendationsPerUser = Integer.parseInt(parsedArgs.get("--numRecommendations"));
- JobConf jobConf = AbstractJob.prepareJobConf(usersFile, outputPath, jarFile, TextInputFormat.class,
+ JobConf jobConf = prepareJobConf(usersFile, outputPath, TextInputFormat.class,
UserIDsMapper.class, LongWritable.class, NullWritable.class, RecommenderReducer.class,
LongWritable.class, RecommendedItemsWritable.class, TextOutputFormat.class);
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java?rev=929484&r1=929483&r2=929484&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java Wed Mar 31 11:47:55 2010
@@ -49,16 +49,15 @@ public final class SlopeOneAverageDiffsJ
String prefsFile = parsedArgs.get("--input");
String outputPath = parsedArgs.get("--output");
- String jarFile = parsedArgs.get("--jarFile");
String averagesOutputPath = parsedArgs.get("--tempDir");
- JobConf prefsToDiffsJobConf = AbstractJob.prepareJobConf(prefsFile, averagesOutputPath, jarFile,
+ JobConf prefsToDiffsJobConf = prepareJobConf(prefsFile, averagesOutputPath,
TextInputFormat.class, ToItemPrefsMapper.class, LongWritable.class, ItemPrefWritable.class,
SlopeOnePrefsToDiffsReducer.class, ItemItemWritable.class, FloatWritable.class,
SequenceFileOutputFormat.class);
JobClient.runJob(prefsToDiffsJobConf);
- JobConf diffsToAveragesJobConf = AbstractJob.prepareJobConf(averagesOutputPath, outputPath, jarFile,
+ JobConf diffsToAveragesJobConf = prepareJobConf(averagesOutputPath, outputPath,
SequenceFileInputFormat.class, IdentityMapper.class, ItemItemWritable.class, FloatWritable.class,
SlopeOneDiffsToAveragesReducer.class, ItemItemWritable.class, FloatWritable.class,
TextOutputFormat.class);
Modified: lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/text/TextParagraphSplittingJob.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/text/TextParagraphSplittingJob.java?rev=929484&r1=929483&r2=929484&view=diff
==============================================================================
--- lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/text/TextParagraphSplittingJob.java (original)
+++ lucene/mahout/trunk/utils/src/main/java/org/apache/mahout/text/TextParagraphSplittingJob.java Wed Mar 31 11:47:55 2010
@@ -41,7 +41,6 @@ public class TextParagraphSplittingJob e
Map<String,String> args = parseArguments(strings);
JobConf conf = prepareJobConf(args.get("--input"),
args.get("--output"),
- args.get("--jarFile"),
SequenceFileInputFormat.class,
SplitMap.class,
Text.class,