You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2010/06/11 13:07:05 UTC
svn commit: r953657 - in /mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/
core/src/main/java/org...
Author: srowen
Date: Fri Jun 11 11:07:05 2010
New Revision: 953657
URL: http://svn.apache.org/viewvc?rev=953657&view=rev
Log:
More MAHOUT-404: use getInputPath(), read --input as well as -Dmapred.input.dir, and make more use of Path along the way versus String
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/DistributedRowMatrix.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/MatrixMultiplicationJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TransposeJob.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java
mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocDriver.java
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/item/RecommenderJob.java Fri Jun 11 11:07:05 2010
@@ -23,7 +23,6 @@ import java.util.concurrent.atomic.Atomi
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.cli2.Option;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
@@ -39,7 +38,6 @@ import org.apache.mahout.cf.taste.hadoop
import org.apache.mahout.cf.taste.hadoop.RecommendedItemsWritable;
import org.apache.mahout.cf.taste.hadoop.ToItemPrefsMapper;
import org.apache.mahout.common.AbstractJob;
-import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.math.VarIntWritable;
import org.apache.mahout.math.VarLongWritable;
import org.apache.mahout.math.VectorWritable;
@@ -73,7 +71,9 @@ public final class RecommenderJob extend
@Override
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
-
+
+ addInputOption();
+ addOutputOption();
addOption("numRecommendations", "n", "Number of recommendations per user",
String.valueOf(AggregateAndRecommendReducer.DEFAULT_NUM_RECOMMENDATIONS));
addOption("usersFile", "u", "File of users to recommend for", null);
@@ -90,9 +90,8 @@ public final class RecommenderJob extend
return -1;
}
- Configuration originalConf = getConf();
- Path inputPath = new Path(originalConf.get("mapred.input.dir"));
- Path outputPath = new Path(originalConf.get("mapred.output.dir"));
+ Path inputPath = getInputPath();
+ Path outputPath = getOutputPath();
Path tempDirPath = new Path(parsedArgs.get("--tempDir"));
int numRecommendations = Integer.parseInt(parsedArgs.get("--numRecommendations"));
String usersFile = parsedArgs.get("--usersFile");
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/pseudo/RecommenderJob.java Fri Jun 11 11:07:05 2010
@@ -106,9 +106,9 @@ public final class RecommenderJob extend
@Override
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
-
- addOption(DefaultOptionCreator.inputOption().create());
- addOption(DefaultOptionCreator.outputOption().create());
+
+ addInputOption();
+ addOutputOption();
addOption("recommenderClassName", "r", "Name of recommender class to instantiate");
addOption("numRecommendations", "n", "Number of recommendations per user", "10");
addOption("usersFile", "u", "Number of recommendations per user", null);
@@ -118,15 +118,9 @@ public final class RecommenderJob extend
return -1;
}
- Configuration originalConf = getConf();
- Path inputFile = new Path(originalConf.get("mapred.input.dir"));
- Path outputPath = new Path(originalConf.get("mapred.output.dir"));
- Path usersFile;
- if (parsedArgs.get("--usersFile") == null) {
- usersFile = inputFile;
- } else {
- usersFile = new Path(parsedArgs.get("--usersFile"));
- }
+ Path inputFile = getInputPath();
+ Path outputPath = getOutputPath();
+ Path usersFile = parsedArgs.get("--usersFile") == null ? inputFile : new Path(parsedArgs.get("--usersFile"));
String recommendClassName = parsedArgs.get("--recommenderClassName");
int recommendationsPerUser = Integer.parseInt(parsedArgs.get("--numRecommendations"));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityJob.java Fri Jun 11 11:07:05 2010
@@ -24,7 +24,6 @@ import java.nio.charset.Charset;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.cli2.Option;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -118,6 +117,8 @@ public final class ItemSimilarityJob ext
@Override
public int run(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
+ addInputOption();
+ addOutputOption();
addOption("similarityClassname", "s", "Name of distributed similarity class to instantiate");
Map<String,String> parsedArgs = parseArguments(args);
@@ -125,12 +126,10 @@ public final class ItemSimilarityJob ext
return -1;
}
- Configuration originalConf = getConf();
-
String distributedSimilarityClassname = parsedArgs.get("--similarityClassname");
- Path inputPath = new Path(originalConf.get("mapred.input.dir"));
- Path outputPath = new Path(originalConf.get("mapred.output.dir"));
+ Path inputPath = getInputPath();
+ Path outputPath = getOutputPath();
Path tempDirPath = new Path(parsedArgs.get("--tempDir"));
Path countUsersPath = new Path(tempDirPath, "countUsers");
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/hadoop/slopeone/SlopeOneAverageDiffsJob.java Fri Jun 11 11:07:05 2010
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.compress.GzipCodec;
@@ -37,7 +36,6 @@ import org.apache.mahout.cf.taste.hadoop
import org.apache.mahout.cf.taste.hadoop.EntityPrefWritable;
import org.apache.mahout.cf.taste.hadoop.ToItemPrefsMapper;
import org.apache.mahout.common.AbstractJob;
-import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.math.VarLongWritable;
public final class SlopeOneAverageDiffsJob extends AbstractJob {
@@ -47,15 +45,14 @@ public final class SlopeOneAverageDiffsJ
addInputOption();
addOutputOption();
-
+
Map<String,String> parsedArgs = parseArguments(args);
if (parsedArgs == null) {
return -1;
}
- Configuration originalConf = getConf();
- Path prefsFile = new Path(originalConf.get("mapred.input.dir"));
- Path outputPath = new Path(originalConf.get("mapred.output.dir"));
+ Path prefsFile = getInputPath();
+ Path outputPath = getOutputPath();
Path averagesOutputPath = new Path(parsedArgs.get("--tempDir"));
AtomicInteger currentPhase = new AtomicInteger();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/AbstractJob.java Fri Jun 11 11:07:05 2010
@@ -24,7 +24,6 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.cli2.Argument;
import org.apache.commons.cli2.CommandLine;
import org.apache.commons.cli2.Group;
import org.apache.commons.cli2.Option;
@@ -92,7 +91,7 @@ public abstract class AbstractJob extend
/** internal list of options that have been added */
private final List<Option> options;
- public AbstractJob() {
+ protected AbstractJob() {
options = new LinkedList<Option>();
}
@@ -102,7 +101,7 @@ public abstract class AbstractJob extend
* property.
* @return
*/
- public Path getInputPath() {
+ protected Path getInputPath() {
return inputPath;
}
@@ -112,7 +111,7 @@ public abstract class AbstractJob extend
* property.
* @return
*/
- public Path getOutputPath() {
+ protected Path getOutputPath() {
return outputPath;
}
@@ -124,7 +123,7 @@ public abstract class AbstractJob extend
* @param shortName
* @param description
*/
- public void addFlag(String name, String shortName, String description) {
+ protected void addFlag(String name, String shortName, String description) {
options.add(buildOption(name, shortName, description, true, false, null));
}
@@ -136,7 +135,7 @@ public abstract class AbstractJob extend
* @param shortName
* @param description
*/
- public void addOption(String name, String shortName, String description) {
+ protected void addOption(String name, String shortName, String description) {
options.add(buildOption(name, shortName, description, true, false, null));
}
@@ -150,7 +149,7 @@ public abstract class AbstractJob extend
* fail with an error and usage message if this option is not specified
* on the command line.
*/
- public void addOption(String name, String shortName, String description, boolean required) {
+ protected void addOption(String name, String shortName, String description, boolean required) {
options.add(buildOption(name, shortName, description, true, required, null));
}
@@ -165,7 +164,7 @@ public abstract class AbstractJob extend
* @param defaultValue the default argument value if this argument is not
* found on the command-line. null is allowed.
*/
- public void addOption(String name, String shortName, String description, String defaultValue) {
+ protected void addOption(String name, String shortName, String description, String defaultValue) {
options.add(buildOption(name, shortName, description, true, false, defaultValue));
}
@@ -178,7 +177,7 @@ public abstract class AbstractJob extend
* @param option
* @return the option added.
*/
- public Option addOption(Option option) {
+ protected Option addOption(Option option) {
options.add(option);
return option;
}
@@ -188,7 +187,7 @@ public abstract class AbstractJob extend
* called, the outputPath will be set based upon the value for this option.
* This this method is called, the output is required.
*/
- public void addInputOption() {
+ protected void addInputOption() {
this.inputOption = addOption(DefaultOptionCreator.inputOption().create());
}
@@ -197,7 +196,7 @@ public abstract class AbstractJob extend
* called, the outputPath will be set based upon the value for this option.
* This this method is called, the output is required.
*/
- public void addOutputOption() {
+ protected void addOutputOption() {
this.outputOption = addOption(DefaultOptionCreator.outputOption().create());
}
@@ -293,7 +292,7 @@ public abstract class AbstractJob extend
}
Map<String,String> result = new TreeMap<String,String>();
- maybePut(result, cmdLine, this.options.toArray(new Option[0]));
+ maybePut(result, cmdLine, this.options.toArray(new Option[this.options.size()]));
parseDirectories(cmdLine);
@@ -310,21 +309,17 @@ public abstract class AbstractJob extend
protected void parseDirectories(CommandLine cmdLine) {
Configuration conf = getConf();
- if (inputOption != null) {
- if (cmdLine.hasOption(inputOption)) {
- this.inputPath = new Path(cmdLine.getValue(inputOption).toString());
- }
+ if (inputOption != null && cmdLine.hasOption(inputOption)) {
+ this.inputPath = new Path(cmdLine.getValue(inputOption).toString());
}
- else if (conf.get("mapred.input.dir") != null) {
+ if (inputPath == null && conf.get("mapred.input.dir") != null) {
this.inputPath = new Path(conf.get("mapred.input.dir"));
}
- if (outputOption != null) {
- if (cmdLine.hasOption(outputOption)) {
- this.outputPath = new Path(cmdLine.getValue(outputOption).toString());
- }
+ if (outputOption != null && cmdLine.hasOption(outputOption)) {
+ this.outputPath = new Path(cmdLine.getValue(outputOption).toString());
}
- else if (conf.get("mapred.output.dir") != null) {
+ if (outputPath == null && conf.get("mapred.output.dir") != null) {
this.outputPath = new Path(conf.get("mapred.output.dir"));
}
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/common/commandline/DefaultOptionCreator.java Fri Jun 11 11:07:05 2010
@@ -41,7 +41,7 @@ public final class DefaultOptionCreator
* Returns a default command line option for input directory specification. Used by all clustering jobs plus others
*/
public static DefaultOptionBuilder inputOption() {
- return new DefaultOptionBuilder().withLongName("input").withRequired(true).withShortName("i").withArgument(
+ return new DefaultOptionBuilder().withLongName("input").withRequired(false).withShortName("i").withArgument(
new ArgumentBuilder().withName("input").withMinimum(1).withMaximum(1).create()).withDescription(
"Path to job input directory. Must be a SequenceFile of VectorWritable");
}
@@ -60,7 +60,7 @@ public final class DefaultOptionCreator
* Returns a default command line option for output directory specification. Used by all clustering jobs plus others
*/
public static DefaultOptionBuilder outputOption() {
- return new DefaultOptionBuilder().withLongName("output").withRequired(true).withShortName("o").withArgument(
+ return new DefaultOptionBuilder().withLongName("output").withRequired(false).withShortName("o").withArgument(
new ArgumentBuilder().withName("output").withMinimum(1).withMaximum(1).create()).withDescription(
"The directory pathname for output.");
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/DistributedRowMatrix.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/DistributedRowMatrix.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/DistributedRowMatrix.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/DistributedRowMatrix.java Fri Jun 11 11:07:05 2010
@@ -63,20 +63,20 @@ public class DistributedRowMatrix implem
private static final Logger log = LoggerFactory.getLogger(DistributedRowMatrix.class);
- private final String inputPathString;
- private final String outputTmpPathString;
+ private final Path inputPath;
+ private final Path outputTmpPath;
private JobConf conf;
private Path rowPath;
private Path outputTmpBasePath;
private final int numRows;
private final int numCols;
- public DistributedRowMatrix(String inputPathString,
- String outputTmpPathString,
+ public DistributedRowMatrix(Path inputPathString,
+ Path outputTmpPathString,
int numRows,
int numCols) {
- this.inputPathString = inputPathString;
- this.outputTmpPathString = outputTmpPathString;
+ this.inputPath = inputPathString;
+ this.outputTmpPath = outputTmpPathString;
this.numRows = numRows;
this.numCols = numCols;
}
@@ -85,8 +85,8 @@ public class DistributedRowMatrix implem
public void configure(JobConf conf) {
this.conf = conf;
try {
- rowPath = FileSystem.get(conf).makeQualified(new Path(inputPathString));
- outputTmpBasePath = FileSystem.get(conf).makeQualified(new Path(outputTmpPathString));
+ rowPath = FileSystem.get(conf).makeQualified(inputPath);
+ outputTmpBasePath = FileSystem.get(conf).makeQualified(outputTmpPath);
} catch (IOException ioe) {
throw new IllegalStateException(ioe);
}
@@ -142,8 +142,7 @@ public class DistributedRowMatrix implem
JobConf conf = MatrixMultiplicationJob.createMatrixMultiplyJobConf(rowPath, other.rowPath, outPath, other.numCols);
try {
JobClient.runJob(conf);
- DistributedRowMatrix out = new DistributedRowMatrix(outPath.toString(),
- outputTmpPathString, numRows, other.numCols());
+ DistributedRowMatrix out = new DistributedRowMatrix(outPath, outputTmpPath, numRows, other.numCols());
out.configure(conf);
return out;
} catch (IOException ioe) {
@@ -156,7 +155,7 @@ public class DistributedRowMatrix implem
try {
JobConf conf = TransposeJob.buildTransposeJobConf(rowPath, outputPath, numRows);
JobClient.runJob(conf);
- DistributedRowMatrix m = new DistributedRowMatrix(outputPath.toString(), outputTmpPathString, numCols, numRows);
+ DistributedRowMatrix m = new DistributedRowMatrix(outputPath, outputTmpPath, numCols, numRows);
m.configure(this.conf);
return m;
} catch (IOException ioe) {
@@ -170,8 +169,8 @@ public class DistributedRowMatrix implem
JobConf conf = TimesSquaredJob.createTimesJobConf(v,
numRows,
rowPath,
- new Path(outputTmpPathString,
- new Path(Long.toString(System.nanoTime()))));
+ new Path(outputTmpPath,
+ Long.toString(System.nanoTime())));
JobClient.runJob(conf);
return TimesSquaredJob.retrieveTimesSquaredOutputVector(conf);
} catch (IOException ioe) {
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/MatrixMultiplicationJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/MatrixMultiplicationJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/MatrixMultiplicationJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/MatrixMultiplicationJob.java Fri Jun 11 11:07:05 2010
@@ -17,7 +17,6 @@
package org.apache.mahout.math.hadoop;
-import org.apache.commons.cli2.Option;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapred.FileOutputFormat;
@@ -83,12 +82,12 @@ public class MatrixMultiplicationJob ext
return -1;
}
- DistributedRowMatrix a = new DistributedRowMatrix(argMap.get("--inputPathA"),
- argMap.get("--tempDir"),
+ DistributedRowMatrix a = new DistributedRowMatrix(new Path(argMap.get("--inputPathA")),
+ new Path(argMap.get("--tempDir")),
Integer.parseInt(argMap.get("--numRowsA")),
Integer.parseInt(argMap.get("--numColsA")));
- DistributedRowMatrix b = new DistributedRowMatrix(argMap.get("--inputPathB"),
- argMap.get("--tempDir"),
+ DistributedRowMatrix b = new DistributedRowMatrix(new Path(argMap.get("--inputPathB")),
+ new Path(argMap.get("--tempDir")),
Integer.parseInt(argMap.get("--numRowsB")),
Integer.parseInt(argMap.get("--numColsB")));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TransposeJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TransposeJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TransposeJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/TransposeJob.java Fri Jun 11 11:07:05 2010
@@ -17,8 +17,6 @@
package org.apache.mahout.math.hadoop;
-import org.apache.commons.cli2.Option;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -55,6 +53,7 @@ public class TransposeJob extends Abstra
@Override
public int run(String[] strings) throws Exception {
+ addInputOption();
addOption("numRows", "nr", "Number of rows of the input matrix");
addOption("numCols", "nc", "Number of columns of the input matrix");
Map<String,String> parsedArgs = parseArguments(strings);
@@ -63,13 +62,12 @@ public class TransposeJob extends Abstra
return 0;
}
- Configuration originalConf = getConf();
- String inputPathString = originalConf.get("mapred.input.dir");
- String outputTmpPathString = parsedArgs.get("--tempDir");
+ Path inputPath = getInputPath();
+ Path outputTmpPath = new Path(parsedArgs.get("--tempDir"));
int numRows = Integer.parseInt(parsedArgs.get("--numRows"));
int numCols = Integer.parseInt(parsedArgs.get("--numCols"));
- DistributedRowMatrix matrix = new DistributedRowMatrix(inputPathString, outputTmpPathString, numRows, numCols);
+ DistributedRowMatrix matrix = new DistributedRowMatrix(inputPath, outputTmpPath, numRows, numCols);
matrix.configure(new JobConf(getConf()));
matrix.transpose();
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/DistributedLanczosSolver.java Fri Jun 11 11:07:05 2010
@@ -67,17 +67,17 @@ public class DistributedLanczosSolver ex
@Override
public int run(String[] strings) throws Exception {
Configuration originalConfig = getConf();
- String inputPathString = originalConfig.get("mapred.input.dir");
- String outputTmpPathString = parsedArgs.get("--tempDir");
+ Path inputPath = new Path(originalConfig.get("mapred.input.dir"));
+ Path outputTmpPath = new Path(parsedArgs.get("--tempDir"));
int numRows = Integer.parseInt(parsedArgs.get("--numRows"));
int numCols = Integer.parseInt(parsedArgs.get("--numCols"));
boolean isSymmetric = Boolean.parseBoolean(parsedArgs.get("--symmetric"));
int desiredRank = Integer.parseInt(parsedArgs.get("--rank"));
- return run(inputPathString, outputTmpPathString, numRows, numCols, isSymmetric, desiredRank);
+ return run(inputPath, outputTmpPath, numRows, numCols, isSymmetric, desiredRank);
}
- public int run(String inputPathString,
- String outputTmpPathString,
+ public int run(Path inputPath,
+ Path outputTmpPath,
int numRows,
int numCols,
boolean isSymmetric,
@@ -87,8 +87,8 @@ public class DistributedLanczosSolver ex
List<Double> eigenValues = new ArrayList<Double>();
String outputEigenVectorPath = originalConfig.get("mapred.output.dir");
- DistributedRowMatrix matrix = new DistributedRowMatrix(inputPathString,
- outputTmpPathString,
+ DistributedRowMatrix matrix = new DistributedRowMatrix(inputPath,
+ outputTmpPath,
numRows,
numCols);
matrix.configure(new JobConf(originalConfig));
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/decomposer/EigenVerificationJob.java Fri Jun 11 11:07:05 2010
@@ -17,12 +17,6 @@
package org.apache.mahout.math.hadoop.decomposer;
-import org.apache.commons.cli2.CommandLine;
-import org.apache.commons.cli2.Group;
-import org.apache.commons.cli2.Option;
-import org.apache.commons.cli2.OptionException;
-import org.apache.commons.cli2.builder.GroupBuilder;
-import org.apache.commons.cli2.commandline.Parser;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -31,7 +25,6 @@ import org.apache.hadoop.io.SequenceFile
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.util.ToolRunner;
import org.apache.mahout.common.AbstractJob;
-import org.apache.mahout.common.CommandLineUtil;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.math.DenseVector;
import org.apache.mahout.math.MatrixSlice;
@@ -85,8 +78,8 @@ public class EigenVerificationJob extend
private double maxError;
private double minEigenValue;
private boolean loadEigensInMemory;
- private String tmpOut;
- private String outPath;
+ private Path tmpOut;
+ private Path outPath;
public void setEigensToVerify(VectorIterable eigens) {
eigensToVerify = eigens;
@@ -100,18 +93,17 @@ public class EigenVerificationJob extend
} else if (argMap.isEmpty()) {
return 0;
}
- Configuration originalConf = getConf();
- outPath = originalConf.get("mapred.output.dir");
- tmpOut = outPath + "/tmp";
+ outPath = getOutputPath();
+ tmpOut = new Path(outPath, "tmp");
if (argMap.get("--eigenInput") != null && eigensToVerify == null) {
- prepareEigens(argMap.get("--eigenInput"), argMap.get("--inMemory") != null);
+ prepareEigens(new Path(argMap.get("--eigenInput")), argMap.get("--inMemory") != null);
}
maxError = Double.parseDouble(argMap.get("--maxError"));
minEigenValue = Double.parseDouble(argMap.get("--minEigenvalue"));
- DistributedRowMatrix c = new DistributedRowMatrix(argMap.get("--corpusInput"), tmpOut, 1, 1);
+ DistributedRowMatrix c = new DistributedRowMatrix(new Path(argMap.get("--corpusInput")), tmpOut, 1, 1);
c.configure(new JobConf(getConf()));
corpus = c;
@@ -132,6 +124,7 @@ public class EigenVerificationJob extend
}
public Map<String,String> handleArgs(String[] args) {
+ addOutputOption();
addOption("eigenInput", "ei",
"The Path for purported eigenVector input files (SequenceFile<WritableComparable,VectorWritable>.", null);
addOption("corpusInput", "ci",
@@ -198,7 +191,7 @@ public class EigenVerificationJob extend
return eigenMetaData;
}
- private void prepareEigens(String eigenInput, boolean inMemory) {
+ private void prepareEigens(Path eigenInput, boolean inMemory) {
DistributedRowMatrix eigens = new DistributedRowMatrix(eigenInput, tmpOut, 1, 1);
eigens.configure(new JobConf(getConf()));
if (inMemory) {
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/TestDistributedRowMatrix.java Fri Jun 11 11:07:05 2010
@@ -147,8 +147,8 @@ public class TestDistributedRowMatrix ex
}
}, true, new Path(baseTmpDirPath, "distMatrix/part-00000"), fs, conf);
- DistributedRowMatrix distMatrix = new DistributedRowMatrix(baseTmpDirPath + "/distMatrix",
- baseTmpDirPath + "/tmpOut",
+ DistributedRowMatrix distMatrix = new DistributedRowMatrix(new Path(baseTmpDirPath, "distMatrix"),
+ new Path(baseTmpDirPath, "tmpOut"),
m.numRows(),
m.numCols());
distMatrix.configure(new JobConf(conf));
Modified: mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocDriver.java
URL: http://svn.apache.org/viewvc/mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocDriver.java?rev=953657&r1=953656&r2=953657&view=diff
==============================================================================
--- mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocDriver.java (original)
+++ mahout/trunk/utils/src/main/java/org/apache/mahout/utils/nlp/collocations/llr/CollocDriver.java Fri Jun 11 11:07:05 2010
@@ -20,10 +20,6 @@ package org.apache.mahout.utils.nlp.coll
import java.io.IOException;
import java.util.Map;
-import org.apache.commons.cli2.OptionException;
-import org.apache.commons.cli2.builder.ArgumentBuilder;
-import org.apache.commons.cli2.builder.DefaultOptionBuilder;
-import org.apache.commons.cli2.builder.GroupBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
@@ -39,7 +35,6 @@ import org.apache.hadoop.mapred.lib.Iden
import org.apache.hadoop.util.ToolRunner;
import org.apache.lucene.analysis.Analyzer;
import org.apache.mahout.common.AbstractJob;
-import org.apache.mahout.common.CommandLineUtil;
import org.apache.mahout.common.HadoopUtil;
import org.apache.mahout.common.commandline.DefaultOptionCreator;
import org.apache.mahout.text.DefaultAnalyzer;