You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by gs...@apache.org on 2008/10/17 19:45:46 UTC
svn commit: r705685 - in /lucene/mahout/trunk: ./ core/ examples/
examples/src/main/java/org/apache/mahout/classifier/bayes/
Author: gsingers
Date: Fri Oct 17 10:45:45 2008
New Revision: 705685
URL: http://svn.apache.org/viewvc?rev=705685&view=rev
Log:
upgraded CLI stuff to use CLI2, various build/pom changes for MAHOUT-69. I believe ant package now works
Modified:
lucene/mahout/trunk/build.xml
lucene/mahout/trunk/common-build.xml
lucene/mahout/trunk/core/pom.xml
lucene/mahout/trunk/examples/pom.xml
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java
lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
Modified: lucene/mahout/trunk/build.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/build.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/build.xml (original)
+++ lucene/mahout/trunk/build.xml Fri Oct 17 10:45:45 2008
@@ -124,7 +124,18 @@
<mahout-checksum file="${dist}/${fullnamever}.zip"/>
<!-- create maven artifacts -->
- <exec command="mvn -Dtest=false deploy"/>
+ <!--<exec command="mvn -Dtest=false deploy"/>-->
+ <input message="Enter the GPG Passphrase" addproperty="gpg.passphrase"/>
+ <exec executable="mvn">
+ <arg value="-Dtest=false"/>
+ <arg value="install"/>
+ <arg value="-Dgpg.passphrase=${gpg.passphrase}"/>
+ </exec>
+ <exec executable="mvn">
+ <arg value="-Dtest=false"/>
+ <arg value="deploy"/>
+ <arg value="-Dgpg.passphrase=${gpg.passphrase}"/>
+ </exec>
<mkdir dir="${dist}/maven"/>
<copy todir="${dist}/maven">
<fileset dir="core/dist/maven"/>
Modified: lucene/mahout/trunk/common-build.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/common-build.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/common-build.xml (original)
+++ lucene/mahout/trunk/common-build.xml Fri Oct 17 10:45:45 2008
@@ -62,7 +62,9 @@
<property name="dist" value="dist" />
- <property name="shared-lib" value="./lib" />
+ <dirname file="${ant.file.common}" property="common.dir"/>
+
+ <property name="shared-lib" value="${common.dir}/lib" />
<!-- Javadoc properties -->
<property name="javadoc.years" value="2008 - ${year}" />
Modified: lucene/mahout/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/pom.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/core/pom.xml (original)
+++ lucene/mahout/trunk/core/pom.xml Fri Oct 17 10:45:45 2008
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-parent</artifactId>
- <version>0.2-SNAPSHOT</version>
+ <version>0.1-SNAPSHOT</version>
</parent>
<groupId>org.apache.mahout</groupId>
Modified: lucene/mahout/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/pom.xml?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/pom.xml (original)
+++ lucene/mahout/trunk/examples/pom.xml Fri Oct 17 10:45:45 2008
@@ -7,7 +7,7 @@
<parent>
<groupId>org.apache.mahout</groupId>
<artifactId>mahout-parent</artifactId>
- <version>0.2-SNAPSHOT</version>
+ <version>0.1-SNAPSHOT</version>
</parent>
<groupId>org.apache.mahout</groupId>
@@ -74,14 +74,14 @@
<dependencies>
<dependency>
- <groupId>org.apache.lucene.mahout</groupId>
+ <groupId>org.apache.mahout</groupId>
<artifactId>core</artifactId>
<version>${parent.version}</version>
</dependency>
<!-- core test -->
<dependency>
- <groupId>org.apache.lucene.mahout</groupId>
+ <groupId>org.apache.mahout</groupId>
<artifactId>core</artifactId>
<version>${parent.version}</version>
<classifier>tests</classifier>
@@ -125,6 +125,18 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jcl</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+
</dependencies>
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/PrepareTwentyNewsgroups.java Fri Oct 17 10:45:45 2008
@@ -16,15 +16,16 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
import org.apache.lucene.analysis.Analyzer;
import org.apache.mahout.classifier.BayesFileFormatter;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import java.io.File;
import java.io.IOException;
@@ -41,25 +42,40 @@
public class PrepareTwentyNewsgroups {
@SuppressWarnings("static-access")
- public static void main(String[] args) throws IOException, ParseException,
- ClassNotFoundException, InstantiationException, IllegalAccessException {
- Options options = new Options();
- Option parentOpt = OptionBuilder.withLongOpt("parent").isRequired().hasArg().withDescription("Parent dir containing the newsgroups").create("p");
- options.addOption(parentOpt);
- Option outputDirOpt = OptionBuilder.withLongOpt("outputDir").isRequired().hasArg().withDescription("The output directory").create("o");
- options.addOption(outputDirOpt);
- Option analyzerNameOpt = OptionBuilder.withLongOpt("analyzerName").isRequired().hasArg().withDescription("The class name of the analyzer").create("a");
- options.addOption(analyzerNameOpt);
- Option charsetOpt = OptionBuilder.withLongOpt("charset").hasArg().isRequired().withDescription("The name of the character encoding of the input files").create("c");
- options.addOption(charsetOpt);
-
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
-
- File parentDir = new File(cmdLine.getOptionValue(parentOpt.getOpt()));
- File outputDir = new File(cmdLine.getOptionValue(outputDirOpt.getOpt()));
- String analyzerName = cmdLine.getOptionValue(analyzerNameOpt.getOpt());
- Charset charset = Charset.forName(cmdLine.getOptionValue(charsetOpt.getOpt()));
+ public static void main(String[] args) throws IOException,
+ ClassNotFoundException, InstantiationException, IllegalAccessException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option parentOpt = obuilder.withLongName("parent").withRequired(true).withArgument(
+ abuilder.withName("parent").withMinimum(1).withMaximum(1).create()).
+ withDescription("Parent dir containing the newsgroups").withShortName("p").create();
+
+ Option outputDirOpt = obuilder.withLongName("outputDir").withRequired(true).withArgument(
+ abuilder.withName("outputDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The output directory").withShortName("o").create();
+
+ Option analyzerNameOpt = obuilder.withLongName("analyzerName").withRequired(true).withArgument(
+ abuilder.withName("analyzerName").withMinimum(1).withMaximum(1).create()).
+ withDescription("The class name of the analyzer").withShortName("a").create();
+
+ Option charsetOpt = obuilder.withLongName("charset").withRequired(true).withArgument(
+ abuilder.withName("charset").withMinimum(1).withMaximum(1).create()).
+ withDescription("The name of the character encoding of the input files").withShortName("c").create();
+
+ Group group = gbuilder.withName("Options").withOption(analyzerNameOpt).withOption(charsetOpt).withOption(outputDirOpt).withOption(parentOpt).create();
+
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+
+
+ File parentDir = new File((String) cmdLine.getValue(parentOpt));
+ File outputDir = new File((String) cmdLine.getValue(outputDirOpt));
+ String analyzerName = (String) cmdLine.getValue(analyzerNameOpt);
+ Charset charset = Charset.forName((String) cmdLine.getValue(charsetOpt));
Analyzer analyzer = (Analyzer) Class.forName(analyzerName).newInstance();
//parent dir contains dir by category
File [] categoryDirs = parentDir.listFiles();
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TestClassifier.java Fri Oct 17 10:45:45 2008
@@ -17,13 +17,6 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
@@ -36,6 +29,14 @@
import org.apache.mahout.classifier.cbayes.CBayesModel;
import org.apache.mahout.common.Classifier;
import org.apache.mahout.common.Model;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,37 +59,53 @@
@SuppressWarnings({ "static-access", "unchecked" })
public static void main(String[] args) throws IOException,
- ClassNotFoundException, IllegalAccessException, InstantiationException, ParseException {
- Options options = new Options();
- Option pathOpt = OptionBuilder.withLongOpt("path").isRequired().hasArg()
- .withDescription("The local file system path").create("p");
- options.addOption(pathOpt);
- Option dirOpt = OptionBuilder.withLongOpt("testDir").isRequired().hasArg()
- .withDescription("The directory where test documents resides in").create("t");
- options.addOption(dirOpt);
- Option encodingOpt = OptionBuilder.withLongOpt("encoding").hasArg()
- .withDescription("The file encoding. defaults to UTF-8").create("e");
- options.addOption(encodingOpt);
- Option analyzerOpt = OptionBuilder.withLongOpt("analyzer").hasArg()
- .withDescription("The Analyzer to use").create("a");
- options.addOption(analyzerOpt);
- Option defaultCatOpt = OptionBuilder.withLongOpt("defaultCat").hasArg()
- .withDescription("The default category").create("d");
- options.addOption(defaultCatOpt);
- Option gramSizeOpt = OptionBuilder.withLongOpt("gramSize").hasArg()
- .withDescription("Size of the n-gram").create("ng");
- options.addOption(gramSizeOpt);
- Option typeOpt = OptionBuilder.withLongOpt("classifierType").isRequired()
- .hasArg().withDescription("Type of classifier").create("type");
- options.addOption(typeOpt);
+ ClassNotFoundException, IllegalAccessException, InstantiationException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option pathOpt = obuilder.withLongName("path").withRequired(true).withArgument(
+ abuilder.withName("path").withMinimum(1).withMaximum(1).create()).
+ withDescription("The local file system path").withShortName("p").create();
+
+ Option dirOpt = obuilder.withLongName("testDir").withRequired(true).withArgument(
+ abuilder.withName("testDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The directory where test documents resides in").withShortName("t").create();
+
+ Option encodingOpt = obuilder.withLongName("encoding").withRequired(true).withArgument(
+ abuilder.withName("encoding").withMinimum(1).withMaximum(1).create()).
+ withDescription("The file encoding. Defaults to UTF-8").withShortName("e").create();
+
+ Option analyzerOpt = obuilder.withLongName("analyzer").withRequired(true).withArgument(
+ abuilder.withName("analyzer").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Analyzer to use").withShortName("a").create();
+
+ Option defaultCatOpt = obuilder.withLongName("defaultCat").withRequired(true).withArgument(
+ abuilder.withName("defaultCat").withMinimum(1).withMaximum(1).create()).
+ withDescription("The default category").withShortName("d").create();
+
+ Option gramSizeOpt = obuilder.withLongName("gramSize").withRequired(true).withArgument(
+ abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).
+ withDescription("Size of the n-gram").withShortName("ng").create();
+
+ Option typeOpt = obuilder.withLongName("classifierType").withRequired(true).withArgument(
+ abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).
+ withDescription("Type of classifier: bayes|cbayes").withShortName("type").create();
+
+ Group group = gbuilder.withName("Options").withOption(analyzerOpt).withOption(defaultCatOpt).withOption(dirOpt).withOption(encodingOpt).withOption(gramSizeOpt).withOption(pathOpt)
+ .withOption(typeOpt).create();
+
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
SequenceFileModelReader reader = new SequenceFileModelReader();
JobConf conf = new JobConf(TestClassifier.class);
Map<String, Path> modelPaths = new HashMap<String, Path>();
- String modelBasePath = cmdLine.getOptionValue(pathOpt.getOpt());
+ String modelBasePath = (String) cmdLine.getValue(pathOpt);
modelPaths.put("sigma_j", new Path(modelBasePath + "/trainer-weights/Sigma_j/part-*"));
modelPaths.put("sigma_k", new Path(modelBasePath + "/trainer-weights/Sigma_k/part-*"));
modelPaths.put("sigma_kSigma_j", new Path(modelBasePath + "/trainer-weights/Sigma_kSigma_j/part-*"));
@@ -102,7 +119,7 @@
Model model;
Classifier classifier;
- String classifierType = cmdLine.getOptionValue(typeOpt.getOpt());
+ String classifierType = (String) cmdLine.getValue(typeOpt);
if (classifierType.equalsIgnoreCase("bayes")) {
log.info("Testing Bayes Classifier");
@@ -123,17 +140,17 @@
log.info("Done generating Model");
String defaultCat = "unknown";
- if (cmdLine.hasOption(defaultCatOpt.getOpt())) {
- defaultCat = cmdLine.getOptionValue(defaultCatOpt.getOpt());
+ if (cmdLine.hasOption(defaultCatOpt)) {
+ defaultCat = (String) cmdLine.getValue(defaultCatOpt);
}
String encoding = "UTF-8";
- if (cmdLine.hasOption(encodingOpt.getOpt())) {
- encoding = cmdLine.getOptionValue(encodingOpt.getOpt());
+ if (cmdLine.hasOption(encodingOpt)) {
+ encoding = (String) cmdLine.getValue(encodingOpt);
}
Analyzer analyzer = null;
- if (cmdLine.hasOption(analyzerOpt.getOpt())) {
- String className = cmdLine.getOptionValue(analyzerOpt.getOpt());
+ if (cmdLine.hasOption(analyzerOpt)) {
+ String className = (String) cmdLine.getValue(analyzerOpt);
Class clazz = Class.forName(className);
analyzer = (Analyzer) clazz.newInstance();
}
@@ -142,13 +159,13 @@
}
// TODO srowen says analyzer is never used?
int gramSize = 1;
- if (cmdLine.hasOption(gramSizeOpt.getOpt())) {
- gramSize = Integer.parseInt(cmdLine
- .getOptionValue(gramSizeOpt.getOpt()));
+ if (cmdLine.hasOption(gramSizeOpt)) {
+ gramSize = Integer.parseInt((String) cmdLine
+ .getValue(gramSizeOpt));
}
- String testDirPath = cmdLine.getOptionValue(dirOpt.getOpt());
+ String testDirPath = (String) cmdLine.getValue(dirOpt);
File dir = new File(testDirPath);
File[] subdirs = dir.listFiles();
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/TrainClassifier.java Fri Oct 17 10:45:45 2008
@@ -16,13 +16,14 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
+import org.apache.commons.cli2.commandline.Parser;
import org.apache.mahout.classifier.cbayes.CBayesDriver;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,7 +32,7 @@
/**
* Train the Naive Bayes Complement classifier with improved weighting on the Twenty Newsgroups data (http://people.csail.mit.edu/jrennie/20Newsgroups/20news-18828.tar.gz)
- *
+ * <p/>
* To run:
* Assume MAHOUT_HOME refers to the location where you checked out/installed Mahout
* <ol>
@@ -40,13 +41,13 @@
* <li>Start up Hadoop and copy the files to the system. See http://hadoop.apache.org/core/docs/r0.16.2/quickstart.html</li>
* <li>From the Hadoop dir (where Hadoop is installed):
* <ol>
- * <li>emacs conf/hadoop-site.xml (add in local settings per quickstart)</li>
- * <li>bin/hadoop namenode -format //Format the HDFS</li>
- * <li>bin/start-all.sh //Start Hadoop</li>
- * <li>bin/hadoop dfs -put <MAHOUT_HOME>/work/20news-18828-collapse 20newsInput //Copies the extracted text to HDFS</li>
- * <li>bin/hadoop jar <MAHOUT_HOME>/build/apache-mahout-0.1-dev-ex.jar org.apache.mahout.examples.classifiers.cbayes.TwentyNewsgroups -t -i 20newsInput -o 20newsOutput</li>
+ * <li>emacs conf/hadoop-site.xml (add in local settings per quickstart)</li>
+ * <li>bin/hadoop namenode -format //Format the HDFS</li>
+ * <li>bin/start-all.sh //Start Hadoop</li>
+ * <li>bin/hadoop dfs -put <MAHOUT_HOME>/work/20news-18828-collapse 20newsInput //Copies the extracted text to HDFS</li>
+ * <li>bin/hadoop jar <MAHOUT_HOME>/build/apache-mahout-0.1-dev-ex.jar org.apache.mahout.examples.classifiers.cbayes.TwentyNewsgroups -t -i 20newsInput -o 20newsOutput</li>
* </ol>
- * </li>
+ * </li>
* </ol>
*/
public class TrainClassifier {
@@ -60,36 +61,47 @@
public void trainCNaiveBayes(String dir, String outputDir, int gramSize) throws IOException {
CBayesDriver.runJob(dir, outputDir, gramSize);
}
-
- @SuppressWarnings("static-access")
- public static void main(String[] args) throws IOException, ParseException {
- Options options = new Options();
- Option trainOpt = OptionBuilder.withLongOpt("train").withDescription("Train the classifier").create("t");
- options.addOption(trainOpt);
- Option inputDirOpt = OptionBuilder.withLongOpt("inputDir").hasArg().withDescription("The Directory on HDFS containing the collapsed, properly formatted files").create("i");
- options.addOption(inputDirOpt);
- Option outputOpt = OptionBuilder.withLongOpt("output").isRequired().hasArg().withDescription("The location of the model on the HDFS").create("o");
- options.addOption(outputOpt);
- Option gramSizeOpt = OptionBuilder.withLongOpt("gramSize").hasArg().withDescription("Size of the n-gram").create("ng");
- options.addOption(gramSizeOpt);
- Option typeOpt = OptionBuilder.withLongOpt("classifierType").isRequired().hasArg().withDescription("Type of classifier").create("type");
- options.addOption(typeOpt);
-
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
- boolean train = cmdLine.hasOption(trainOpt.getOpt());
+ @SuppressWarnings("static-access")
+ public static void main(String[] args) throws IOException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+ Option trainOpt = obuilder.withLongName("train").withRequired(true).
+ withDescription("Train the classifier").withShortName("t").create();
+
+ Option inputDirOpt = obuilder.withLongName("inputDir").withRequired(true).withArgument(
+ abuilder.withName("inputDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Directory on HDFS containing the collapsed, properly formatted files").withShortName("i").create();
+
+ Option outputOpt = obuilder.withLongName("output").withRequired(true).withArgument(
+ abuilder.withName("output").withMinimum(1).withMaximum(1).create()).
+ withDescription("The location of the modelon the HDFS").withShortName("o").create();
+
+ Option gramSizeOpt = obuilder.withLongName("gramSize").withRequired(true).withArgument(
+ abuilder.withName("gramSize").withMinimum(1).withMaximum(1).create()).
+ withDescription("Size of the n-gram").withShortName("ng").create();
+
+ Option typeOpt = obuilder.withLongName("classifierType").withRequired(true).withArgument(
+ abuilder.withName("classifierType").withMinimum(1).withMaximum(1).create()).
+ withDescription("Type of classifier: bayes or cbayes").withShortName("type").create();
+ Group group = gbuilder.withName("Options").withOption(gramSizeOpt).withOption(inputDirOpt).withOption(outputOpt).withOption(trainOpt).withOption(typeOpt).create();
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+ boolean train = cmdLine.hasOption(trainOpt);
TrainClassifier tn = new TrainClassifier();
- if (train){
- String classifierType = cmdLine.getOptionValue(typeOpt.getOpt());
- if(classifierType.equalsIgnoreCase("bayes")){
+ if (train) {
+ String classifierType = (String) cmdLine.getValue(typeOpt);
+ if (classifierType.equalsIgnoreCase("bayes")) {
log.info("Training Bayes Classifier");
- tn.trainNaiveBayes(cmdLine.getOptionValue(inputDirOpt.getOpt()), cmdLine.getOptionValue(outputOpt.getOpt()), Integer.parseInt(cmdLine.getOptionValue(gramSizeOpt.getOpt())));
+ tn.trainNaiveBayes((String)cmdLine.getValue(inputDirOpt), (String)cmdLine.getValue(outputOpt), Integer.parseInt((String) cmdLine.getValue(gramSizeOpt)));
- } else if(classifierType.equalsIgnoreCase("cbayes")) {
+ } else if (classifierType.equalsIgnoreCase("cbayes")) {
log.info("Training Complementary Bayes Classifier");
//setup the HDFS and copy the files there, then run the trainer
- tn.trainCNaiveBayes(cmdLine.getOptionValue(inputDirOpt.getOpt()), cmdLine.getOptionValue(outputOpt.getOpt()), Integer.parseInt(cmdLine.getOptionValue(gramSizeOpt.getOpt())));
+ tn.trainCNaiveBayes((String) cmdLine.getValue(inputDirOpt), (String) cmdLine.getValue(outputOpt), Integer.parseInt((String) cmdLine.getValue(gramSizeOpt)));
}
}
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaDatasetCreator.java Fri Oct 17 10:45:45 2008
@@ -17,13 +17,15 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
+
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import java.io.IOException;
@@ -31,24 +33,35 @@
@SuppressWarnings("static-access")
public static void main(String[] args) throws IOException,
- ClassNotFoundException, IllegalAccessException, InstantiationException, ParseException {
- Options options = new Options();
- Option dirInputPathOpt = OptionBuilder.withLongOpt("dirInputPath").isRequired().hasArg()
- .withDescription("The input Directory Path").create("i");
- options.addOption(dirInputPathOpt);
- Option dirOutputPathOpt = OptionBuilder.withLongOpt("dirOuputPath").isRequired().hasArg()
- .withDescription("The output Directory Path").create("o");
- options.addOption(dirOutputPathOpt);
- Option countriesFileOpt = OptionBuilder.withLongOpt("countriesFile").isRequired().hasArg()
- .withDescription("Location of the Countries File").create("c");
- options.addOption(countriesFileOpt);
+ ClassNotFoundException, IllegalAccessException, InstantiationException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option dirInputPathOpt = obuilder.withLongName("dirInputPath").withRequired(true).withArgument(
+ abuilder.withName("dirInputPath").withMinimum(1).withMaximum(1).create()).
+ withDescription("The input directory path").withShortName("i").create();
+
+ Option dirOutputPathOpt = obuilder.withLongName("dirOutputPath").withRequired(true).withArgument(
+ abuilder.withName("dirOutputPath").withMinimum(1).withMaximum(1).create()).
+ withDescription("The output directory Path").withShortName("o").create();
+
+ Option countriesFileOpt = obuilder.withLongName("countriesFile").withRequired(true).withArgument(
+ abuilder.withName("countriesFile").withMinimum(1).withMaximum(1).create()).
+ withDescription("Location of the countries file").withShortName("c").create();
+
+ Group group = gbuilder.withName("Options").withOption(countriesFileOpt).withOption(dirInputPathOpt).withOption(dirOutputPathOpt).create();
+
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
- String dirInputPath = cmdLine.getOptionValue(dirInputPathOpt.getOpt());
- String dirOutputPath = cmdLine.getOptionValue(dirOutputPathOpt.getOpt());
- String countriesFile = cmdLine.getOptionValue(countriesFileOpt.getOpt());
+
+ String dirInputPath = (String) cmdLine.getValue(dirInputPathOpt);
+ String dirOutputPath = (String) cmdLine.getValue(dirOutputPathOpt);
+ String countriesFile = (String) cmdLine.getValue(countriesFileOpt);
WikipediaDatasetCreatorDriver.runJob(dirInputPath, dirOutputPath, countriesFile);
Modified: lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java?rev=705685&r1=705684&r2=705685&view=diff
==============================================================================
--- lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java (original)
+++ lucene/mahout/trunk/examples/src/main/java/org/apache/mahout/classifier/bayes/WikipediaXmlSplitter.java Fri Oct 17 10:45:45 2008
@@ -17,13 +17,15 @@
* limitations under the License.
*/
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.PosixParser;
-import org.apache.commons.cli.ParseException;
-import org.apache.commons.cli.Parser;
+
+import org.apache.commons.cli2.Option;
+import org.apache.commons.cli2.CommandLine;
+import org.apache.commons.cli2.Group;
+import org.apache.commons.cli2.OptionException;
+import org.apache.commons.cli2.commandline.Parser;
+import org.apache.commons.cli2.builder.DefaultOptionBuilder;
+import org.apache.commons.cli2.builder.ArgumentBuilder;
+import org.apache.commons.cli2.builder.GroupBuilder;
import java.io.BufferedReader;
import java.io.BufferedWriter;
@@ -39,21 +41,31 @@
public class WikipediaXmlSplitter {
@SuppressWarnings("static-access")
- public static void main(String[] args) throws IOException, ParseException {
- Options options = new Options();
- Option dumpFileOpt = OptionBuilder.withLongOpt("dumpfile").isRequired().hasArg().withDescription("The path to the wikipedia dump file").create("d");
- options.addOption(dumpFileOpt);
- Option outputDirOpt = OptionBuilder.withLongOpt("outputDir").isRequired().hasArg().withDescription("The output directory to place the splits in").create("o");
- options.addOption(outputDirOpt);
- Option chunkSizeOpt = OptionBuilder.withLongOpt("chunkSize").isRequired().hasArg().withDescription("the Size of chunk in Megabytes").create("c");
- options.addOption(chunkSizeOpt);
-
- Parser parser = new PosixParser();
- CommandLine cmdLine = parser.parse(options, args);
-
- String dumpFilePath = cmdLine.getOptionValue(dumpFileOpt.getOpt());
- String outputDirPath = cmdLine.getOptionValue(outputDirOpt.getOpt());
- int chunkSize = 1024 * 1024 * Integer.parseInt(cmdLine.getOptionValue(chunkSizeOpt.getOpt()));
+ public static void main(String[] args) throws IOException, OptionException {
+ final DefaultOptionBuilder obuilder = new DefaultOptionBuilder();
+ final ArgumentBuilder abuilder = new ArgumentBuilder();
+ final GroupBuilder gbuilder = new GroupBuilder();
+
+ Option dumpFileOpt = obuilder.withLongName("dumpFile").withRequired(true).withArgument(
+ abuilder.withName("dumpFile").withMinimum(1).withMaximum(1).create()).
+ withDescription("The path to the wikipedia dump file").withShortName("d").create();
+
+ Option outputDirOpt = obuilder.withLongName("outputDir").withRequired(true).withArgument(
+ abuilder.withName("outputDir").withMinimum(1).withMaximum(1).create()).
+ withDescription("The output directory to place the splits in").withShortName("o").create();
+
+ Option chunkSizeOpt = obuilder.withLongName("chunkSize").withRequired(true).withArgument(
+ abuilder.withName("chunkSize").withMinimum(1).withMaximum(1).create()).
+ withDescription("The Size of the chunk, in megabytes").withShortName("c").create();
+ Group group = gbuilder.withName("Options").withOption(dumpFileOpt).withOption(outputDirOpt).withOption(chunkSizeOpt).create();
+ CommandLine cmdLine;
+ Parser parser = new Parser();
+ parser.setGroup(group);
+ cmdLine = parser.parse(args);
+
+ String dumpFilePath = (String) cmdLine.getValue(dumpFileOpt);
+ String outputDirPath = (String) cmdLine.getValue(outputDirOpt);
+ int chunkSize = 1024 * 1024 * Integer.parseInt((String) cmdLine.getValue(chunkSizeOpt));
BufferedReader dumpReader = new BufferedReader(new InputStreamReader(
new FileInputStream(dumpFilePath), "UTF-8"));