You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/08/23 19:00:04 UTC
svn commit: r1160801 - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/cmdline/tokenizer/
main/java/opennlp/tools/tokenize/ test/java/opennlp/tools/tokenize/
Author: colen
Date: Tue Aug 23 17:00:04 2011
New Revision: 1160801
URL: http://svn.apache.org/viewvc?rev=1160801&view=rev
Log:
OPENNLP-226 Tokenizer Monitors are passed as variable length param.
Added:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluationMonitor.java (with props)
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerEvaluatorTest.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java?rev=1160801&r1=1160800&r2=1160801&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java Tue Aug 23 17:00:04 2011
@@ -21,6 +21,7 @@ import java.io.OutputStream;
import opennlp.tools.cmdline.EvaluationErrorPrinter;
import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.tokenize.TokenizerEvaluationMonitor;
import opennlp.tools.util.eval.EvaluationMonitor;
/**
@@ -29,7 +30,7 @@ import opennlp.tools.util.eval.Evaluatio
*
*/
public class TokenEvaluationErrorListener extends
- EvaluationErrorPrinter<TokenSample> {
+ EvaluationErrorPrinter<TokenSample> implements TokenizerEvaluationMonitor {
/**
* Creates a listener that will print to System.err
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java?rev=1160801&r1=1160800&r2=1160801&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java Tue Aug 23 17:00:04 2011
@@ -30,9 +30,9 @@ import opennlp.tools.cmdline.params.CVPa
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.tokenize.TokenizerCrossValidator;
+import opennlp.tools.tokenize.TokenizerEvaluationMonitor;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
-import opennlp.tools.util.eval.EvaluationMonitor;
import opennlp.tools.util.eval.FMeasure;
public final class TokenizerCrossValidatorTool implements CmdLineTool {
@@ -81,7 +81,7 @@ public final class TokenizerCrossValidat
mlParams = TokenizerTrainerTool.createTrainingParameters(
params.getIterations(), params.getCutoff());
- EvaluationMonitor<TokenSample> listener = null;
+ TokenizerEvaluationMonitor listener = null;
if (params.getMisclassified()) {
listener = new TokenEvaluationErrorListener();
}
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java?rev=1160801&r1=1160800&r2=1160801&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java Tue Aug 23 17:00:04 2011
@@ -20,7 +20,6 @@ package opennlp.tools.cmdline.tokenizer;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
-import java.util.Collections;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CLI;
@@ -29,10 +28,10 @@ import opennlp.tools.cmdline.CmdLineUtil
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.cmdline.params.EvaluatorParams;
import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.tokenize.TokenizerEvaluationMonitor;
import opennlp.tools.tokenize.TokenizerEvaluator;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.eval.EvaluationMonitor;
public final class TokenizerMEEvaluatorTool implements CmdLineTool {
@@ -62,13 +61,13 @@ public final class TokenizerMEEvaluatorT
TokenizerModel model = new TokenizerModelLoader().load(params.getModel());
- EvaluationMonitor<TokenSample> missclassifiedListener = null;
+ TokenizerEvaluationMonitor missclassifiedListener = null;
if (params.getMisclassified()) {
missclassifiedListener = new TokenEvaluationErrorListener();
}
TokenizerEvaluator evaluator = new TokenizerEvaluator(
- new opennlp.tools.tokenize.TokenizerME(model), Collections.singletonList(missclassifiedListener));
+ new opennlp.tools.tokenize.TokenizerME(model), missclassifiedListener);
System.out.print("Evaluating ... ");
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java?rev=1160801&r1=1160800&r2=1160801&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java Tue Aug 23 17:00:04 2011
@@ -18,15 +18,11 @@
package opennlp.tools.tokenize;
import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
import opennlp.tools.dictionary.Dictionary;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
-import opennlp.tools.util.eval.EvaluationMonitor;
import opennlp.tools.util.eval.FMeasure;
import opennlp.tools.util.model.ModelUtil;
@@ -40,7 +36,7 @@ public class TokenizerCrossValidator {
private final Dictionary abbreviations;
private FMeasure fmeasure = new FMeasure();
- private LinkedList<EvaluationMonitor<TokenSample>> listeners;
+ private TokenizerEvaluationMonitor[] listeners;
public TokenizerCrossValidator(String language, boolean alphaNumericOptimization, int cutoff, int iterations) {
@@ -49,51 +45,22 @@ public class TokenizerCrossValidator {
public TokenizerCrossValidator(String language, boolean alphaNumericOptimization) {
this(language, alphaNumericOptimization, ModelUtil.createTrainingParameters(100, 5));
- }
-
- public TokenizerCrossValidator(String language, boolean alphaNumericOptimization, TrainingParameters params) {
- this(language, null, alphaNumericOptimization, params);
- }
-
- public TokenizerCrossValidator(String language, Dictionary abbreviations,
- boolean alphaNumericOptimization, TrainingParameters params) {
-
- this.language = language;
- this.alphaNumericOptimization = alphaNumericOptimization;
- this.abbreviations = abbreviations;
- this.params = params;
-
}
public TokenizerCrossValidator(String language,
boolean alphaNumericOptimization, TrainingParameters params,
- List<? extends EvaluationMonitor<TokenSample>> listeners) {
+ TokenizerEvaluationMonitor ... listeners) {
this(language, null, alphaNumericOptimization, params, listeners);
}
public TokenizerCrossValidator(String language, Dictionary abbreviations,
boolean alphaNumericOptimization, TrainingParameters params,
- List<? extends EvaluationMonitor<TokenSample>> listeners) {
-
- this(language, abbreviations, alphaNumericOptimization, params);
- if (listeners != null) {
- this.listeners = new LinkedList<EvaluationMonitor<TokenSample>>(
- listeners);
- }
- }
-
- public TokenizerCrossValidator(String language,
- boolean alphaNumericOptimization, TrainingParameters params,
- EvaluationMonitor<TokenSample> listener) {
- this(language, null, alphaNumericOptimization, params, Collections
- .singletonList(listener));
- }
-
- public TokenizerCrossValidator(String language, Dictionary abbreviations,
- boolean alphaNumericOptimization, TrainingParameters params,
- EvaluationMonitor<TokenSample> listener) {
- this(language, abbreviations, alphaNumericOptimization, params, Collections
- .singletonList(listener));
+ TokenizerEvaluationMonitor ... listeners) {
+ this.language = language;
+ this.alphaNumericOptimization = alphaNumericOptimization;
+ this.abbreviations = abbreviations;
+ this.params = params;
+ this.listeners = listeners;
}
/**
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluationMonitor.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluationMonitor.java?rev=1160801&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluationMonitor.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluationMonitor.java Tue Aug 23 17:00:04 2011
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.tokenize;
+
+import opennlp.tools.util.eval.EvaluationMonitor;
+
+public interface TokenizerEvaluationMonitor extends
+ EvaluationMonitor<TokenSample> {
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluationMonitor.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java?rev=1160801&r1=1160800&r2=1160801&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java Tue Aug 23 17:00:04 2011
@@ -18,11 +18,7 @@
package opennlp.tools.tokenize;
-import java.util.Collections;
-import java.util.List;
-
import opennlp.tools.util.Span;
-import opennlp.tools.util.eval.EvaluationMonitor;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.FMeasure;
@@ -44,16 +40,6 @@ public class TokenizerEvaluator extends
* predicted tokens.
*/
private Tokenizer tokenizer;
-
- /**
- * Initializes the current instance with the
- * given {@link Tokenizer}.
- *
- * @param tokenizer the {@link Tokenizer} to evaluate.
- */
- public TokenizerEvaluator(Tokenizer tokenizer) {
- this.tokenizer = tokenizer;
- }
/**
* Initializes the current instance with the
@@ -62,23 +48,10 @@ public class TokenizerEvaluator extends
* @param tokenizer the {@link Tokenizer} to evaluate.
* @param listeners evaluation sample listeners
*/
- public TokenizerEvaluator(Tokenizer tokenizer, List<? extends EvaluationMonitor<TokenSample>> listeners) {
+ public TokenizerEvaluator(Tokenizer tokenizer, TokenizerEvaluationMonitor ... listeners) {
super(listeners);
this.tokenizer = tokenizer;
}
-
- /**
- * Initializes the current instance with the given {@link Tokenizer}.
- *
- * @param tokenizer
- * the {@link Tokenizer} to evaluate.
- * @param listener
- * evaluation sample listener
- */
- public TokenizerEvaluator(Tokenizer tokenizer,
- EvaluationMonitor<TokenSample> listener) {
- this(tokenizer, Collections.singletonList(listener));
- }
@Override
protected TokenSample processSample(TokenSample reference) {
Modified: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerEvaluatorTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerEvaluatorTest.java?rev=1160801&r1=1160800&r2=1160801&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerEvaluatorTest.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerEvaluatorTest.java Tue Aug 23 17:00:04 2011
@@ -22,12 +22,10 @@ import static junit.framework.Assert.ass
import java.io.ByteArrayOutputStream;
import java.io.OutputStream;
-import java.util.Collections;
import opennlp.tools.cmdline.tokenizer.TokenEvaluationErrorListener;
import opennlp.tools.util.InvalidFormatException;
import opennlp.tools.util.Span;
-import opennlp.tools.util.eval.EvaluationMonitor;
import org.junit.Test;
@@ -36,11 +34,11 @@ public class TokenizerEvaluatorTest {
@Test
public void testPositive() throws InvalidFormatException {
OutputStream stream = new ByteArrayOutputStream();
- EvaluationMonitor<TokenSample> listener = new TokenEvaluationErrorListener(
+ TokenizerEvaluationMonitor listener = new TokenEvaluationErrorListener(
stream);
TokenizerEvaluator eval = new TokenizerEvaluator(new DummyTokenizer(
- TokenSampleTest.createGoldSample()), Collections.singletonList(listener));
+ TokenSampleTest.createGoldSample()), listener);
eval.evaluateSample(TokenSampleTest.createGoldSample());
@@ -52,11 +50,11 @@ public class TokenizerEvaluatorTest {
@Test
public void testNegative() throws InvalidFormatException {
OutputStream stream = new ByteArrayOutputStream();
- EvaluationMonitor<TokenSample> listener = new TokenEvaluationErrorListener(
+ TokenizerEvaluationMonitor listener = new TokenEvaluationErrorListener(
stream);
TokenizerEvaluator eval = new TokenizerEvaluator(new DummyTokenizer(
- TokenSampleTest.createGoldSample()), Collections.singletonList(listener));
+ TokenSampleTest.createGoldSample()), listener);
eval.evaluateSample(TokenSampleTest.createPredSample());