You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/07/18 21:33:59 UTC
svn commit: r1148012 - in
/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser:
ParserTrainerTool.java TrainingParameters.java TrainingParams.java
Author: colen
Date: Mon Jul 18 19:33:58 2011
New Revision: 1148012
URL: http://svn.apache.org/viewvc?rev=1148012&view=rev
Log:
OPENNLP-227 Updated Parser trainer tool
Added:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParams.java (with props)
Removed:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParameters.java
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java?rev=1148012&r1=1148011&r2=1148012&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/ParserTrainerTool.java Mon Jul 18 19:33:58 2011
@@ -26,10 +26,12 @@ import java.io.InputStreamReader;
import java.nio.charset.Charset;
import opennlp.model.TrainUtil;
+import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.cmdline.TrainingToolParams;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.parser.HeadRules;
import opennlp.tools.parser.Parse;
@@ -41,6 +43,10 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.PlainTextByLineStream;
public final class ParserTrainerTool implements CmdLineTool {
+
+ interface TrainerToolParams extends TrainingParams, TrainingToolParams{
+
+ }
public String getName() {
return "ParserTrainer";
@@ -51,8 +57,8 @@ public final class ParserTrainerTool imp
}
public String getHelp() {
- return "Usage: " + CLI.CMD + " " + getName() + " " + TrainingParameters.getParameterUsage() +
- " -head-rules head_rules -data trainingData -model model\n" + TrainingParameters.getDescription();
+ return "Usage: " + CLI.CMD + " " + getName() + " "
+ + ArgumentParser.createUsage(TrainerToolParams.class);
}
static ObjectStream<Parse> openTrainingData(File trainingDataFile, Charset encoding) {
@@ -93,23 +99,32 @@ public final class ParserTrainerTool imp
return mdict;
}
+ static ParserType parseParserType(String typeAsString) {
+ ParserType type = null;
+ if(typeAsString != null && typeAsString.length() > 0) {
+ type = ParserType.parse(typeAsString);
+ if(type == null) {
+ System.err.println("ParserType training parameter is invalid!");
+ throw new TerminateToolException(-1);
+ }
+ }
+
+ return type;
+ }
+
// TODO: Add param to train tree insert parser
public void run(String[] args) {
- if (args.length < 10) {
- System.out.println(getHelp());
+ if (!ArgumentParser.validateArguments(args, TrainerToolParams.class)) {
+ System.err.println(getHelp());
throw new TerminateToolException(1);
}
-
- TrainingParameters parameters = new TrainingParameters(args);
- if(!parameters.isValid()) {
- System.out.println(getHelp());
- throw new TerminateToolException(1);
- }
+ TrainerToolParams params = ArgumentParser.parse(args,
+ TrainerToolParams.class);
opennlp.tools.util.TrainingParameters mlParams =
- CmdLineUtil.loadTrainingParameters(CmdLineUtil.getParameter("-params", args), true);
+ CmdLineUtil.loadTrainingParameters(params.getParams(), true);
if (mlParams != null) {
if (!TrainUtil.isValid(mlParams.getSettings("build"))) {
@@ -138,40 +153,42 @@ public final class ParserTrainerTool imp
}
}
- ObjectStream<Parse> sampleStream = openTrainingData(new File(CmdLineUtil.getParameter("-data", args)), parameters.getEncoding());
+ ObjectStream<Parse> sampleStream = openTrainingData(params.getData(), params.getEncoding());
- File modelOutFile = new File(CmdLineUtil.getParameter("-model", args));
+ File modelOutFile = params.getModel();
CmdLineUtil.checkOutputFile("parser model", modelOutFile);
ParserModel model;
try {
HeadRules rules = new opennlp.tools.parser.lang.en.HeadRules(
- new InputStreamReader(new FileInputStream(new File(CmdLineUtil.getParameter("-head-rules", args))),
- parameters.getEncoding()));
+ new InputStreamReader(new FileInputStream(params.getHeadRules()),
+ params.getEncoding()));
+
+ ParserType type = parseParserType(params.getParserType());
if (mlParams == null) {
- if (ParserType.CHUNKING.equals(parameters.getParserType())) {
+ if (ParserType.CHUNKING.equals(type)) {
model = opennlp.tools.parser.chunking.Parser.train(
- parameters.getLanguage(), sampleStream, rules,
- parameters.getNumberOfIterations(), parameters.getCutoff());
+ params.getLang(), sampleStream, rules,
+ params.getIterations(), params.getCutoff());
}
- else if (ParserType.TREEINSERT.equals(parameters.getParserType())) {
- model = opennlp.tools.parser.treeinsert.Parser.train(parameters.getLanguage(), sampleStream, rules, parameters.getNumberOfIterations(),
- parameters.getCutoff());
+ else if (ParserType.TREEINSERT.equals(type)) {
+ model = opennlp.tools.parser.treeinsert.Parser.train(params.getLang(), sampleStream, rules, params.getIterations(),
+ params.getCutoff());
}
else {
throw new IllegalStateException();
}
}
else {
- if (ParserType.CHUNKING.equals(parameters.getParserType())) {
+ if (ParserType.CHUNKING.equals(type)) {
model = opennlp.tools.parser.chunking.Parser.train(
- parameters.getLanguage(), sampleStream, rules,
+ params.getLang(), sampleStream, rules,
mlParams);
}
- else if (ParserType.TREEINSERT.equals(parameters.getParserType())) {
- model = opennlp.tools.parser.treeinsert.Parser.train(parameters.getLanguage(), sampleStream, rules,
+ else if (ParserType.TREEINSERT.equals(type)) {
+ model = opennlp.tools.parser.treeinsert.Parser.train(params.getLang(), sampleStream, rules,
mlParams);
}
else {
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParams.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParams.java?rev=1148012&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParams.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParams.java Mon Jul 18 19:33:58 2011
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.parser;
+
+import java.io.File;
+
+import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
+import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
+import opennlp.tools.cmdline.BasicTrainingParams;
+
+/**
+ * TrainingParams for Parser.
+ *
+ * Note: Do not use this class, internal use only!
+ */
+interface TrainingParams extends BasicTrainingParams {
+
+ @ParameterDescription(valueName = "CHUNKING|TREEINSERT", description = "One of CHUNKING or TREEINSERT. Default is CHUNKING.")
+ @OptionalParameter(defaultValue = "CHUNKING")
+ String getParserType();
+
+
+ @ParameterDescription(valueName = "headRulesFile", description = "the head rules file")
+ File getHeadRules();
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/parser/TrainingParams.java
------------------------------------------------------------------------------
svn:mime-type = text/plain