You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/08/17 17:03:29 UTC
svn commit: r1158760 [1/2] - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/chunker/ main/java/opennlp/tools/cmdline/
main/java/opennlp/tools/cmdline/chunker/
main/java/opennlp/tools/cmdline/namefind/ main/java/opennlp/tools/cmd...
Author: colen
Date: Wed Aug 17 15:03:28 2011
New Revision: 1158760
URL: http://svn.apache.org/viewvc?rev=1158760&view=rev
Log:
OPENNLP-226 Evaluators now allow tools to register a misclassified report interface
Added:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkEvaluationErrorListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/NameEvaluationErrorListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSEvaluationErrorListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceEvaluationErrorListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/MissclassifiedSampleListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/postag/POSEvaluatorTest.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/sentdetect/SentenceDetectorEvaluatorTest.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/tokenize/TokenizerEvaluatorTest.java (with props)
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEvaluator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorEvaluator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/Evaluator.java
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerCrossValidator.java Wed Aug 17 15:03:28 2011
@@ -24,6 +24,7 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public class ChunkerCrossValidator {
@@ -63,7 +64,7 @@ public class ChunkerCrossValidator {
*/
public void evaluate(ObjectStream<ChunkSample> samples, int nFolds)
throws IOException, InvalidFormatException, IOException {
- evaluate(samples, nFolds, false);
+ evaluate(samples, nFolds, null);
}
/**
@@ -73,13 +74,13 @@ public class ChunkerCrossValidator {
* the data to train and test
* @param nFolds
* number of folds
- * @param printErrors
- * if true will print errors
+ * @param listener
+ * an optional missclassified sample listener
*
* @throws IOException
*/
public void evaluate(ObjectStream<ChunkSample> samples, int nFolds,
- boolean printErrors) throws IOException, InvalidFormatException,
+ MissclassifiedSampleListener<ChunkSample> listener) throws IOException, InvalidFormatException,
IOException {
CrossValidationPartitioner<ChunkSample> partitioner = new CrossValidationPartitioner<ChunkSample>(
samples, nFolds);
@@ -101,7 +102,7 @@ public class ChunkerCrossValidator {
}
// do testing
- ChunkerEvaluator evaluator = new ChunkerEvaluator(new ChunkerME(model), printErrors);
+ ChunkerEvaluator evaluator = new ChunkerEvaluator(new ChunkerME(model), listener);
evaluator.evaluate(trainingSampleStream.getTestSampleStream());
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEvaluator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEvaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/chunker/ChunkerEvaluator.java Wed Aug 17 15:03:28 2011
@@ -20,6 +20,7 @@ package opennlp.tools.chunker;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
/**
* The {@link ChunkerEvaluator} measures the performance
@@ -40,6 +41,8 @@ public class ChunkerEvaluator extends Ev
*/
private Chunker chunker;
+ private MissclassifiedSampleListener<ChunkSample> sampleListener;
+
/**
* Initializes the current instance with the given
* {@link Chunker}.
@@ -55,11 +58,13 @@ public class ChunkerEvaluator extends Ev
* {@link Chunker}.
*
* @param chunker the {@link Chunker} to evaluate.
- * @param printError outputs errors
+ * @param sampleListener
+ * an optional {@link MissclassifiedSampleListener} listener to
+ * notify errors
*/
- public ChunkerEvaluator(Chunker chunker, boolean printError) {
- super(printError);
+ public ChunkerEvaluator(Chunker chunker, MissclassifiedSampleListener<ChunkSample> sampleListener) {
this.chunker = chunker;
+ this.sampleListener = sampleListener;
}
/**
@@ -77,9 +82,12 @@ public class ChunkerEvaluator extends Ev
String[] preds = chunker.chunk(reference.getSentence(), reference.getTags());
ChunkSample result = new ChunkSample(reference.getSentence(), reference.getTags(), preds);
- if (isPrintError()) {
- String[] sentence = reference.getSentence();
- printErrors(reference.getPreds(), preds, reference, result, sentence);
+ if (this.sampleListener != null) {
+ ChunkSample predicted = new ChunkSample(reference.getSentence(), reference.getTags(),
+ preds);
+ if (!predicted.equals(reference)) {
+ this.sampleListener.missclassified(reference, predicted);
+ }
}
fmeasure.updateScores(reference.getPhrasesAsSpanList(), result.getPhrasesAsSpanList());
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,217 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline;
+
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import opennlp.tools.util.Span;
+
+public class EvaluationErrorPrinter<T> {
+
+ private PrintStream printStream;
+
+ protected EvaluationErrorPrinter(OutputStream outputStream) {
+ this.printStream = new PrintStream(outputStream);
+ }
+
+ // for the sentence detector
+ protected void printError(Span references[], Span predictions[],
+ T referenceSample, T predictedSample, String sentence) {
+ List<Span> falseNegatives = new ArrayList<Span>();
+ List<Span> falsePositives = new ArrayList<Span>();
+
+ findErrors(references, predictions, falseNegatives, falsePositives);
+
+ if (falsePositives.size() + falseNegatives.size() > 0) {
+
+ printSamples(referenceSample, predictedSample);
+
+ printErrors(falsePositives, falseNegatives, sentence);
+
+ }
+ }
+
+ // for namefinder, chunker...
+ protected void printError(Span references[], Span predictions[],
+ T referenceSample, T predictedSample, String[] sentenceTokens) {
+ List<Span> falseNegatives = new ArrayList<Span>();
+ List<Span> falsePositives = new ArrayList<Span>();
+
+ findErrors(references, predictions, falseNegatives, falsePositives);
+
+ if (falsePositives.size() + falseNegatives.size() > 0) {
+
+ printSamples(referenceSample, predictedSample);
+
+ printErrors(falsePositives, falseNegatives, sentenceTokens);
+
+ }
+ }
+
+ // for pos tagger
+ protected void printError(String references[], String predictions[],
+ T referenceSample, T predictedSample, String[] sentenceTokens) {
+ List<String> filteredDoc = new ArrayList<String>();
+ List<String> filteredRefs = new ArrayList<String>();
+ List<String> filteredPreds = new ArrayList<String>();
+
+ for (int i = 0; i < references.length; i++) {
+ if (!references[i].equals(predictions[i])) {
+ filteredDoc.add(sentenceTokens[i]);
+ filteredRefs.add(references[i]);
+ filteredPreds.add(predictions[i]);
+ }
+ }
+
+ if (filteredDoc.size() > 0) {
+
+ printSamples(referenceSample, predictedSample);
+
+ printErrors(filteredDoc, filteredRefs, filteredPreds);
+
+ }
+ }
+
+ /**
+ * Auxiliary method to print tag errors
+ *
+ * @param filteredDoc
+ * the document tokens which were tagged wrong
+ * @param filteredRefs
+ * the reference tags
+ * @param filteredPreds
+ * the predicted tags
+ */
+ private void printErrors(List<String> filteredDoc, List<String> filteredRefs,
+ List<String> filteredPreds) {
+ printStream.println("Errors: {");
+ printStream.println("Tok: Ref | Pred");
+ printStream.println("---------------");
+ for (int i = 0; i < filteredDoc.size(); i++) {
+ printStream.println(filteredDoc.get(i) + ": " + filteredRefs.get(i)
+ + " | " + filteredPreds.get(i));
+ }
+ printStream.println("}\n");
+ }
+
+ /**
+ * Auxiliary method to print span errors
+ *
+ * @param falsePositives
+ * false positives span
+ * @param falseNegatives
+ * false negative span
+ * @param doc
+ * the document text
+ */
+ private void printErrors(List<Span> falsePositives,
+ List<Span> falseNegatives, String doc) {
+ printStream.println("False positives: {");
+ for (Span span : falsePositives) {
+ printStream.println(span.getCoveredText(doc));
+ }
+ printStream.println("} False negatives: {");
+ for (Span span : falseNegatives) {
+ printStream.println(span.getCoveredText(doc));
+ }
+ printStream.println("}\n");
+ }
+
+ /**
+ * Auxiliary method to print span errors
+ *
+ * @param falsePositives
+ * false positives span
+ * @param falseNegatives
+ * false negative span
+ * @param toks
+ * the document tokens
+ */
+ private void printErrors(List<Span> falsePositives,
+ List<Span> falseNegatives, String[] toks) {
+ printStream.println("False positives: {");
+ printStream.println(print(falsePositives, toks));
+ printStream.println("} False negatives: {");
+ printStream.println(print(falseNegatives, toks));
+ printStream.println("}\n");
+ }
+
+ /**
+ * Auxiliary method to print spans
+ *
+ * @param spans
+ * the span list
+ * @param toks
+ * the tokens array
+ * @return the spans as string
+ */
+ private String print(List<Span> spans, String[] toks) {
+ return Arrays.toString(Span.spansToStrings(
+ spans.toArray(new Span[spans.size()]), toks));
+ }
+
+ /**
+ * Auxiliary method to print expected and predicted samples.
+ *
+ * @param referenceSample
+ * the reference sample
+ * @param predictedSample
+ * the predicted sample
+ */
+ private <S> void printSamples(S referenceSample, S predictedSample) {
+ String details = "Expected: {\n" + referenceSample + "}\nPredicted: {\n"
+ + predictedSample + "}";
+ printStream.println(details);
+ }
+
+ /**
+ * Outputs falseNegatives and falsePositives spans from the references and
+ * predictions list.
+ *
+ * @param references
+ * @param predictions
+ * @param falseNegatives
+ * [out] the false negatives list
+ * @param falsePositives
+ * [out] the false positives list
+ */
+ private void findErrors(Span references[], Span predictions[],
+ List<Span> falseNegatives, List<Span> falsePositives) {
+
+ falseNegatives.addAll(Arrays.asList(references));
+ falsePositives.addAll(Arrays.asList(predictions));
+
+ for (int referenceIndex = 0; referenceIndex < references.length; referenceIndex++) {
+
+ Span referenceName = references[referenceIndex];
+
+ for (int predictedIndex = 0; predictedIndex < predictions.length; predictedIndex++) {
+ if (referenceName.equals(predictions[predictedIndex])) {
+ // got it, remove from fn and fp
+ falseNegatives.remove(referenceName);
+ falsePositives.remove(predictions[predictedIndex]);
+ }
+ }
+ }
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/EvaluationErrorPrinter.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkEvaluationErrorListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkEvaluationErrorListener.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkEvaluationErrorListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkEvaluationErrorListener.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.chunker;
+
+import java.io.OutputStream;
+
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.cmdline.EvaluationErrorPrinter;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
+
+/**
+ * A default implementation of {@link MissclassifiedSampleListener} that prints
+ * to an output stream.
+ *
+ */
+public class ChunkEvaluationErrorListener extends
+ EvaluationErrorPrinter<ChunkSample> implements
+ MissclassifiedSampleListener<ChunkSample> {
+
+ /**
+ * Creates a listener that will print to System.err
+ */
+ public ChunkEvaluationErrorListener() {
+ super(System.err);
+ }
+
+ /**
+ * Creates a listener that will print to a given {@link OutputStream}
+ */
+ public ChunkEvaluationErrorListener(OutputStream outputStream) {
+ super(outputStream);
+ }
+
+ public void missclassified(ChunkSample reference, ChunkSample prediction) {
+ printError(reference.getPhrasesAsSpanList(),
+ prediction.getPhrasesAsSpanList(), reference, prediction,
+ reference.getSentence());
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkEvaluationErrorListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java Wed Aug 17 15:03:28 2011
@@ -30,6 +30,7 @@ import opennlp.tools.cmdline.CmdLineUtil
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class ChunkerCrossValidatorTool implements CmdLineTool {
@@ -68,9 +69,14 @@ public final class ChunkerCrossValidator
ChunkerCrossValidator validator = new ChunkerCrossValidator(
params.getLang(), params.getCutoff(), params.getIterations());
+
+ MissclassifiedSampleListener<ChunkSample> errorListener = null;
+ if(params.getMisclassified()) {
+ errorListener = new ChunkEvaluationErrorListener();
+ }
try {
- validator.evaluate(sampleStream, params.getFolds(), params.getMisclassified());
+ validator.evaluate(sampleStream, params.getFolds(), errorListener);
}
catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java Wed Aug 17 15:03:28 2011
@@ -33,6 +33,7 @@ import opennlp.tools.cmdline.CmdLineUtil
import opennlp.tools.cmdline.PerformanceMonitor;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class ChunkerEvaluatorTool implements CmdLineTool {
@@ -64,8 +65,13 @@ public final class ChunkerEvaluatorTool
Charset encoding = params.getEncoding();
ChunkerModel model = new ChunkerModelLoader().load(params.getModel());
+
+ MissclassifiedSampleListener<ChunkSample> errorListener = null;
+ if(params.getMisclassified()) {
+ errorListener = new ChunkEvaluationErrorListener();
+ }
- ChunkerEvaluator evaluator = new ChunkerEvaluator(new ChunkerME(model), params.getMisclassified());
+ ChunkerEvaluator evaluator = new ChunkerEvaluator(new ChunkerME(model), errorListener);
final ObjectStream<ChunkSample> sampleStream = ChunkerTrainerTool.openSampleData("Test",
testData, encoding);
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/NameEvaluationErrorListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/NameEvaluationErrorListener.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/NameEvaluationErrorListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/NameEvaluationErrorListener.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.namefind;
+
+import java.io.OutputStream;
+
+import opennlp.tools.cmdline.EvaluationErrorPrinter;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
+
+/**
+ * A default implementation of {@link MissclassifiedSampleListener} that prints
+ * to an output stream.
+ *
+ */
+public class NameEvaluationErrorListener extends
+ EvaluationErrorPrinter<NameSample> implements
+ MissclassifiedSampleListener<NameSample> {
+
+ /**
+ * Creates a listener that will print to System.err
+ */
+ public NameEvaluationErrorListener() {
+ super(System.err);
+ }
+
+ /**
+ * Creates a listener that will print to a given {@link OutputStream}
+ */
+ public NameEvaluationErrorListener(OutputStream outputStream) {
+ super(outputStream);
+ }
+
+ public void missclassified(NameSample reference, NameSample prediction) {
+ printError(reference.getNames(), prediction.getNames(), reference,
+ prediction, reference.getSentence());
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/NameEvaluationErrorListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java Wed Aug 17 15:03:28 2011
@@ -31,6 +31,7 @@ import opennlp.tools.cmdline.TerminateTo
import opennlp.tools.namefind.NameSample;
import opennlp.tools.namefind.TokenNameFinderCrossValidator;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class TokenNameFinderCrossValidatorTool implements CmdLineTool {
@@ -77,6 +78,11 @@ public final class TokenNameFinderCrossV
.openSampleData("Training Data", trainingDataInFile, encoding);
TokenNameFinderCrossValidator validator;
+
+ MissclassifiedSampleListener<NameSample> errorListener = null;
+ if (params.getMisclassified()) {
+ errorListener = new NameEvaluationErrorListener();
+ }
try {
if (mlParams == null) {
@@ -87,7 +93,7 @@ public final class TokenNameFinderCrossV
validator = new TokenNameFinderCrossValidator(params.getLang(), params.getType(), mlParams,
featureGeneratorBytes, resources);
}
- validator.evaluate(sampleStream, params.getFolds(), params.getMisclassified());
+ validator.evaluate(sampleStream, params.getFolds(), errorListener);
} catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
throw new TerminateToolException(-1);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java Wed Aug 17 15:03:28 2011
@@ -32,6 +32,7 @@ import opennlp.tools.namefind.NameFinder
import opennlp.tools.namefind.NameSample;
import opennlp.tools.namefind.TokenNameFinderModel;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class TokenNameFinderEvaluatorTool implements CmdLineTool {
@@ -66,9 +67,14 @@ public final class TokenNameFinderEvalua
TokenNameFinderModel model = new TokenNameFinderModelLoader().load(params
.getModel());
+
+ MissclassifiedSampleListener<NameSample> missclassifiedListener = null;
+ if (params.getMisclassified()) {
+ missclassifiedListener = new NameEvaluationErrorListener();
+ }
opennlp.tools.namefind.TokenNameFinderEvaluator evaluator = new opennlp.tools.namefind.TokenNameFinderEvaluator(
- new NameFinderME(model), params.getMisclassified());
+ new NameFinderME(model), missclassifiedListener);
final ObjectStream<NameSample> sampleStream = TokenNameFinderTrainerTool.openSampleData("Test",
testData, encoding);
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSEvaluationErrorListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSEvaluationErrorListener.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSEvaluationErrorListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSEvaluationErrorListener.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.postag;
+
+import java.io.OutputStream;
+
+import opennlp.tools.cmdline.EvaluationErrorPrinter;
+import opennlp.tools.postag.POSSample;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
+
+/**
+ * A default implementation of {@link MissclassifiedSampleListener} that prints
+ * to an output stream.
+ *
+ */
+public class POSEvaluationErrorListener extends
+ EvaluationErrorPrinter<POSSample> implements
+ MissclassifiedSampleListener<POSSample> {
+
+ /**
+ * Creates a listener that will print to System.err
+ */
+ public POSEvaluationErrorListener() {
+ super(System.err);
+ }
+
+ /**
+ * Creates a listener that will print to a given {@link OutputStream}
+ */
+ public POSEvaluationErrorListener(OutputStream outputStream) {
+ super(outputStream);
+ }
+
+ public void missclassified(POSSample reference, POSSample prediction) {
+ printError(reference.getTags(), prediction.getTags(), reference,
+ prediction, reference.getSentence());
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSEvaluationErrorListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerCrossValidatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerCrossValidatorTool.java Wed Aug 17 15:03:28 2011
@@ -31,6 +31,7 @@ import opennlp.tools.postag.POSDictionar
import opennlp.tools.postag.POSSample;
import opennlp.tools.postag.POSTaggerCrossValidator;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class POSTaggerCrossValidatorTool implements CmdLineTool {
@@ -84,8 +85,13 @@ public final class POSTaggerCrossValidat
validator = new POSTaggerCrossValidator(params.getLang(),
mlParams, tagdict, null);
}
+
+ MissclassifiedSampleListener<POSSample> missclassifiedListener = null;
+ if (params.getMisclassified()) {
+ missclassifiedListener = new POSEvaluationErrorListener();
+ }
- validator.evaluate(sampleStream, params.getFolds(), params.getMisclassified());
+ validator.evaluate(sampleStream, params.getFolds(), missclassifiedListener);
} catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
throw new TerminateToolException(-1);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerEvaluatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/postag/POSTaggerEvaluatorTool.java Wed Aug 17 15:03:28 2011
@@ -31,6 +31,7 @@ import opennlp.tools.postag.POSEvaluator
import opennlp.tools.postag.POSModel;
import opennlp.tools.postag.POSSample;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class POSTaggerEvaluatorTool implements CmdLineTool {
@@ -63,10 +64,15 @@ public final class POSTaggerEvaluatorToo
Charset encoding = params.getEncoding();
POSModel model = new POSModelLoader().load(params.getModel());
-
- POSEvaluator evaluator =
- new POSEvaluator(new opennlp.tools.postag.POSTaggerME(model), params.getMisclassified());
-
+
+ MissclassifiedSampleListener<POSSample> missclassifiedListener = null;
+ if (params.getMisclassified()) {
+ missclassifiedListener = new POSEvaluationErrorListener();
+ }
+
+ POSEvaluator evaluator = new POSEvaluator(
+ new opennlp.tools.postag.POSTaggerME(model), missclassifiedListener);
+
System.out.print("Evaluating ... ");
ObjectStream<POSSample> sampleStream =
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorCrossValidatorTool.java Wed Aug 17 15:03:28 2011
@@ -32,6 +32,7 @@ import opennlp.tools.sentdetect.SDCrossV
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class SentenceDetectorCrossValidatorTool implements CmdLineTool {
@@ -72,6 +73,11 @@ public final class SentenceDetectorCross
trainingDataInFile, encoding);
SDCrossValidator validator;
+
+ MissclassifiedSampleListener<SentenceSample> errorListener = null;
+ if (params.getMisclassified()) {
+ errorListener = new SentenceEvaluationErrorListener();
+ }
try {
Dictionary abbreviations = SentenceDetectorTrainerTool.loadDict(
@@ -84,7 +90,7 @@ public final class SentenceDetectorCross
abbreviations);
}
- validator.evaluate(sampleStream, params.getFolds(), params.getMisclassified());
+ validator.evaluate(sampleStream, params.getFolds(), errorListener);
}
catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorEvaluatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceDetectorEvaluatorTool.java Wed Aug 17 15:03:28 2011
@@ -22,15 +22,17 @@ import java.io.IOException;
import java.nio.charset.Charset;
import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
+import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.sentdetect.SentenceDetectorEvaluator;
import opennlp.tools.sentdetect.SentenceDetectorME;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.sentdetect.SentenceSample;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class SentenceDetectorEvaluatorTool implements CmdLineTool {
@@ -62,8 +64,13 @@ public final class SentenceDetectorEvalu
File trainingDataInFile = params.getData();
CmdLineUtil.checkInputFile("Training Data", trainingDataInFile);
- opennlp.tools.sentdetect.SentenceDetectorEvaluator evaluator = new opennlp.tools.sentdetect.SentenceDetectorEvaluator(
- new SentenceDetectorME(model), params.getMisclassified());
+ MissclassifiedSampleListener<SentenceSample> errorListener = null;
+ if (params.getMisclassified()) {
+ errorListener = new SentenceEvaluationErrorListener();
+ }
+
+ SentenceDetectorEvaluator evaluator = new SentenceDetectorEvaluator(
+ new SentenceDetectorME(model), errorListener);
System.out.print("Evaluating ... ");
ObjectStream<SentenceSample> sampleStream = SentenceDetectorTrainerTool.openSampleData("Test",
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceEvaluationErrorListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceEvaluationErrorListener.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceEvaluationErrorListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceEvaluationErrorListener.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.sentdetect;
+
+import java.io.OutputStream;
+
+import opennlp.tools.cmdline.EvaluationErrorPrinter;
+import opennlp.tools.sentdetect.SentenceSample;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
+
+/**
+ * A default implementation of {@link MissclassifiedSampleListener} that prints
+ * to an output stream.
+ *
+ */
+public class SentenceEvaluationErrorListener extends
+ EvaluationErrorPrinter<SentenceSample> implements
+ MissclassifiedSampleListener<SentenceSample> {
+
+ /**
+ * Creates a listener that will print to System.err
+ */
+ public SentenceEvaluationErrorListener() {
+ super(System.err);
+ }
+
+ /**
+ * Creates a listener that will print to a given {@link OutputStream}
+ */
+ public SentenceEvaluationErrorListener(OutputStream outputStream) {
+ super(outputStream);
+ }
+
+ public void missclassified(SentenceSample reference, SentenceSample prediction) {
+ printError(reference.getSentences(), prediction.getSentences(), reference,
+ prediction, reference.getDocument());
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/sentdetect/SentenceEvaluationErrorListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.tokenizer;
+
+import java.io.OutputStream;
+
+import opennlp.tools.cmdline.EvaluationErrorPrinter;
+import opennlp.tools.tokenize.TokenSample;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
+
+/**
+ * A default implementation of {@link MissclassifiedSampleListener} that prints
+ * to an output stream.
+ *
+ */
+public class TokenEvaluationErrorListener extends
+ EvaluationErrorPrinter<TokenSample> implements
+ MissclassifiedSampleListener<TokenSample> {
+
+ /**
+ * Creates a listener that will print to System.err
+ */
+ public TokenEvaluationErrorListener() {
+ super(System.err);
+ }
+
+ /**
+ * Creates a listener that will print to a given {@link OutputStream}
+ */
+ public TokenEvaluationErrorListener(OutputStream outputStream) {
+ super(outputStream);
+ }
+
+ public void missclassified(TokenSample reference, TokenSample prediction) {
+ printError(reference.getTokenSpans(), prediction.getTokenSpans(),
+ reference, prediction, reference.getText());
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenEvaluationErrorListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerCrossValidatorTool.java Wed Aug 17 15:03:28 2011
@@ -32,6 +32,7 @@ import opennlp.tools.tokenize.TokenSampl
import opennlp.tools.tokenize.TokenizerCrossValidator;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class TokenizerCrossValidatorTool implements CmdLineTool {
@@ -78,6 +79,11 @@ public final class TokenizerCrossValidat
if (mlParams == null)
mlParams = TokenizerTrainerTool.createTrainingParameters(
params.getIterations(), params.getCutoff());
+
+ MissclassifiedSampleListener<TokenSample> missclassifiedListener = null;
+ if (params.getMisclassified()) {
+ missclassifiedListener = new TokenEvaluationErrorListener();
+ }
try {
Dictionary dict = TokenizerTrainerTool.loadDict(params.getAbbDict(), params.getIsAbbDictCS());
@@ -85,7 +91,7 @@ public final class TokenizerCrossValidat
validator = new opennlp.tools.tokenize.TokenizerCrossValidator(
params.getLang(), dict, params.getAlphaNumOpt(), mlParams);
- validator.evaluate(sampleStream, params.getFolds(), params.getMisclassified());
+ validator.evaluate(sampleStream, params.getFolds(), missclassifiedListener);
}
catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/tokenizer/TokenizerMEEvaluatorTool.java Wed Aug 17 15:03:28 2011
@@ -22,15 +22,16 @@ import java.io.IOException;
import java.nio.charset.Charset;
import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
+import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.tokenize.TokenSample;
import opennlp.tools.tokenize.TokenizerEvaluator;
import opennlp.tools.tokenize.TokenizerModel;
import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public final class TokenizerMEEvaluatorTool implements CmdLineTool {
@@ -60,8 +61,13 @@ public final class TokenizerMEEvaluatorT
TokenizerModel model = new TokenizerModelLoader().load(params.getModel());
+ MissclassifiedSampleListener<TokenSample> missclassifiedListener = null;
+ if (params.getMisclassified()) {
+ missclassifiedListener = new TokenEvaluationErrorListener();
+ }
+
TokenizerEvaluator evaluator = new TokenizerEvaluator(
- new opennlp.tools.tokenize.TokenizerME(model), params.getMisclassified());
+ new opennlp.tools.tokenize.TokenizerME(model), missclassifiedListener);
System.out.print("Evaluating ... ");
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderCrossValidator.java Wed Aug 17 15:03:28 2011
@@ -25,6 +25,7 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public class TokenNameFinderCrossValidator {
@@ -144,7 +145,7 @@ public class TokenNameFinderCrossValidat
*/
public void evaluate(ObjectStream<NameSample> samples, int nFolds)
throws IOException {
- evaluate(samples, nFolds, false);
+ evaluate(samples, nFolds, null);
}
/**
@@ -152,12 +153,12 @@ public class TokenNameFinderCrossValidat
*
* @param samples the data to train and test
* @param nFolds number of folds
- * @param printErrors if true will print errors
+ * @param listener an optional listener to print missclassified items
*
* @throws IOException
*/
- public void evaluate(ObjectStream<NameSample> samples, int nFolds, boolean printErrors)
- throws IOException {
+ public void evaluate(ObjectStream<NameSample> samples, int nFolds,
+ MissclassifiedSampleListener<NameSample> listener) throws IOException {
CrossValidationPartitioner<NameSample> partitioner = new CrossValidationPartitioner<NameSample>(
samples, nFolds);
@@ -177,7 +178,7 @@ public class TokenNameFinderCrossValidat
// do testing
TokenNameFinderEvaluator evaluator = new TokenNameFinderEvaluator(
- new NameFinderME(model), printErrors);
+ new NameFinderME(model), listener);
evaluator.evaluate(trainingSampleStream.getTestSampleStream());
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/TokenNameFinderEvaluator.java Wed Aug 17 15:03:28 2011
@@ -30,6 +30,7 @@ import opennlp.tools.util.PlainTextByLin
import opennlp.tools.util.Span;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
/**
* The {@link TokenNameFinderEvaluator} measures the performance
@@ -50,25 +51,30 @@ public class TokenNameFinderEvaluator ex
*/
private TokenNameFinder nameFinder;
+ private MissclassifiedSampleListener<NameSample> sampleListener;
+
/**
* Initializes the current instance with the given
* {@link TokenNameFinder}.
*
* @param nameFinder the {@link TokenNameFinder} to evaluate.
*/
- public TokenNameFinderEvaluator(TokenNameFinder nameFinder, boolean printErrors) {
- super(printErrors);
+ public TokenNameFinderEvaluator(TokenNameFinder nameFinder) {
this.nameFinder = nameFinder;
}
/**
- * Initializes the current instance with the given
- * {@link TokenNameFinder}.
- *
- * @param nameFinder the {@link TokenNameFinder} to evaluate.
+ * Initializes the current instance with the given {@link TokenNameFinder}.
+ *
+ * @param nameFinder
+ * the {@link TokenNameFinder} to evaluate.
+ * @param sampleListener
+ * an optional {@link MissclassifiedSampleListener} listener to
+ * notify errors
*/
- public TokenNameFinderEvaluator(TokenNameFinder nameFinder) {
+ public TokenNameFinderEvaluator(TokenNameFinder nameFinder, MissclassifiedSampleListener<NameSample> sampleListener) {
this.nameFinder = nameFinder;
+ this.sampleListener = sampleListener;
}
/**
@@ -85,11 +91,13 @@ public class TokenNameFinderEvaluator ex
Span predictedNames[] = nameFinder.find(reference.getSentence());
Span references[] = reference.getNames();
-
- if (isPrintError()) {
- String[] sentence = reference.getSentence();
- printErrors(references, predictedNames, reference, new NameSample(sentence,
- predictedNames, true), sentence);
+
+ if (this.sampleListener != null) {
+ NameSample predicted = new NameSample(reference.getSentence(), predictedNames,
+ reference.isClearAdaptiveDataSet());
+ if(!predicted.equals(reference)) {
+ this.sampleListener.missclassified(reference, predicted);
+ }
}
fmeasure.updateScores(references, predictedNames);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSEvaluator.java Wed Aug 17 15:03:28 2011
@@ -20,6 +20,7 @@ package opennlp.tools.postag;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.Mean;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
/**
* The {@link POSEvaluator} measures the performance of
@@ -32,6 +33,8 @@ public class POSEvaluator extends Evalua
private Mean wordAccuracy = new Mean();
+ private MissclassifiedSampleListener<POSSample> sampleListener;
+
/**
* Initializes the current instance.
*
@@ -45,11 +48,13 @@ public class POSEvaluator extends Evalua
* Initializes the current instance.
*
* @param tagger
- * @param printErrors
+ * @param sampleListener
+ * an optional {@link MissclassifiedSampleListener} listener to
+ * notify errors
*/
- public POSEvaluator(POSTagger tagger, boolean printErrors) {
- super(printErrors);
+ public POSEvaluator(POSTagger tagger, MissclassifiedSampleListener<POSSample> sampleListener) {
this.tagger = tagger;
+ this.sampleListener = sampleListener;
}
/**
@@ -66,10 +71,11 @@ public class POSEvaluator extends Evalua
String predictedTags[] = tagger.tag(reference.getSentence());
String referenceTags[] = reference.getTags();
- if (isPrintError()) {
- String[] sentence = reference.getSentence();
- printErrors(referenceTags, predictedTags, reference, new POSSample(sentence,
- predictedTags), sentence);
+ if (this.sampleListener != null) {
+ POSSample predicted = new POSSample(reference.getSentence(), predictedTags);
+ if(!predicted.equals(reference)) {
+ this.sampleListener.missclassified(reference, predicted);
+ }
}
for (int i = 0; i < referenceTags.length; i++) {
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/postag/POSTaggerCrossValidator.java Wed Aug 17 15:03:28 2011
@@ -24,6 +24,7 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
import opennlp.tools.util.eval.Mean;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
import opennlp.tools.util.model.ModelType;
public class POSTaggerCrossValidator {
@@ -81,7 +82,7 @@ public class POSTaggerCrossValidator {
*/
public void evaluate(ObjectStream<POSSample> samples, int nFolds)
throws IOException, IOException {
- evaluate(samples, nFolds, false);
+ evaluate(samples, nFolds, null);
}
/**
@@ -91,13 +92,13 @@ public class POSTaggerCrossValidator {
* the data to train and test
* @param nFolds
* number of folds
- * @param printErrors
- * if true will print errors
+ * @param listener
+ * an optional listener to print missclassified items
*
* @throws IOException
*/
public void evaluate(ObjectStream<POSSample> samples, int nFolds,
- boolean printErrors) throws IOException, IOException {
+ MissclassifiedSampleListener<POSSample> listener) throws IOException, IOException {
CrossValidationPartitioner<POSSample> partitioner = new CrossValidationPartitioner<POSSample>(
samples, nFolds);
@@ -117,7 +118,7 @@ public class POSTaggerCrossValidator {
this.tagDictionary, this.ngramDictionary);
}
- POSEvaluator evaluator = new POSEvaluator(new POSTaggerME(model), printErrors);
+ POSEvaluator evaluator = new POSEvaluator(new POSTaggerME(model), listener);
evaluator.evaluate(trainingSampleStream.getTestSampleStream());
wordAccuracy.add(evaluator.getWordAccuracy(), evaluator.getWordCount());
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SDCrossValidator.java Wed Aug 17 15:03:28 2011
@@ -24,6 +24,7 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
/**
*
@@ -81,9 +82,9 @@ public class SDCrossValidator {
*/
public void evaluate(ObjectStream<SentenceSample> samples, int nFolds)
throws IOException {
- evaluate(samples, nFolds, false);
+ evaluate(samples, nFolds, null);
}
-
+
/**
* Starts the evaluation.
*
@@ -91,13 +92,13 @@ public class SDCrossValidator {
* the data to train and test
* @param nFolds
* number of folds
- * @param printErrors
- * if true will print errors
+ * @param listener
+ * an optional listener to print missclassified items
*
* @throws IOException
*/
public void evaluate(ObjectStream<SentenceSample> samples, int nFolds,
- boolean printErrors) throws IOException {
+ MissclassifiedSampleListener<SentenceSample> listener) throws IOException {
CrossValidationPartitioner<SentenceSample> partitioner =
new CrossValidationPartitioner<SentenceSample>(samples, nFolds);
@@ -114,7 +115,7 @@ public class SDCrossValidator {
// do testing
SentenceDetectorEvaluator evaluator = new SentenceDetectorEvaluator(
- new SentenceDetectorME(model), printErrors);
+ new SentenceDetectorME(model), listener);
evaluator.evaluate(trainingSampleStream.getTestSampleStream());
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorEvaluator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorEvaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/sentdetect/SentenceDetectorEvaluator.java Wed Aug 17 15:03:28 2011
@@ -20,6 +20,7 @@ package opennlp.tools.sentdetect;
import opennlp.tools.util.Span;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
/**
* The {@link SentenceDetectorEvaluator} measures the performance of
@@ -39,15 +40,20 @@ public class SentenceDetectorEvaluator e
*/
private SentenceDetector sentenceDetector;
+ private MissclassifiedSampleListener<SentenceSample> sampleListener;
+
/**
* Initializes the current instance.
*
* @param sentenceDetector
- * @param isPrintErrors if should print false positives and negatives
+ * @param sampleListener
+ * an optional {@link MissclassifiedSampleListener} listener to
+ * notify errors
*/
- public SentenceDetectorEvaluator(SentenceDetector sentenceDetector, boolean isPrintErrors) {
- super(isPrintErrors);
+ public SentenceDetectorEvaluator(SentenceDetector sentenceDetector,
+ MissclassifiedSampleListener<SentenceSample> sampleListener) {
this.sentenceDetector = sentenceDetector;
+ this.sampleListener = sampleListener;
}
/**
@@ -63,10 +69,11 @@ public class SentenceDetectorEvaluator e
Span predictions[] = sentenceDetector.sentPosDetect(sample.getDocument());
Span[] references = sample.getSentences();
- if (isPrintError()) {
- String doc = sample.getDocument();
- printErrors(references, predictions, sample, new SentenceSample(doc,
- predictions), doc);
+ if (this.sampleListener != null) {
+ SentenceSample predicted = new SentenceSample(sample.getDocument(), predictions);
+ if(!predicted.equals(sample)) {
+ this.sampleListener.missclassified(sample, predicted);
+ }
}
fmeasure.updateScores(references, predictions);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerCrossValidator.java Wed Aug 17 15:03:28 2011
@@ -24,6 +24,7 @@ import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.TrainingParameters;
import opennlp.tools.util.eval.CrossValidationPartitioner;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
public class TokenizerCrossValidator {
@@ -72,7 +73,7 @@ public class TokenizerCrossValidator {
*/
public void evaluate(ObjectStream<TokenSample> samples, int nFolds)
throws IOException {
- evaluate(samples, nFolds, false);
+ evaluate(samples, nFolds, null);
}
/**
@@ -82,13 +83,13 @@ public class TokenizerCrossValidator {
* the data to train and test
* @param nFolds
* number of folds
- * @param printErrors
- * if true will print errors
+ * @param listener
+ * an optional listener to print missclassified items
*
* @throws IOException
*/
public void evaluate(ObjectStream<TokenSample> samples, int nFolds,
- boolean printErrors) throws IOException {
+ MissclassifiedSampleListener<TokenSample> listener) throws IOException {
CrossValidationPartitioner<TokenSample> partitioner =
new CrossValidationPartitioner<TokenSample>(samples, nFolds);
@@ -104,7 +105,7 @@ public class TokenizerCrossValidator {
model = TokenizerME.train(language, trainingSampleStream, abbreviations,
alphaNumericOptimization, params);
- TokenizerEvaluator evaluator = new TokenizerEvaluator(new TokenizerME(model), printErrors);
+ TokenizerEvaluator evaluator = new TokenizerEvaluator(new TokenizerME(model), listener);
evaluator.evaluate(trainingSampleStream.getTestSampleStream());
fmeasure.mergeInto(evaluator.getFMeasure());
}
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/tokenize/TokenizerEvaluator.java Wed Aug 17 15:03:28 2011
@@ -21,6 +21,7 @@ package opennlp.tools.tokenize;
import opennlp.tools.util.Span;
import opennlp.tools.util.eval.Evaluator;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
/**
* The {@link TokenizerEvaluator} measures the performance of
@@ -40,17 +41,21 @@ public class TokenizerEvaluator extends
* predicted tokens.
*/
private Tokenizer tokenizer;
+
+ private MissclassifiedSampleListener<TokenSample> sampleListener;
/**
* Initializes the current instance with the
* given {@link Tokenizer}.
*
* @param tokenizer the {@link Tokenizer} to evaluate.
- * @param printError should print detailed output
+ * @param sampleListener
+ * an optional {@link MissclassifiedSampleListener} listener to
+ * notify errors
*/
- public TokenizerEvaluator(Tokenizer tokenizer, boolean printErrors) {
- super(printErrors);
+ public TokenizerEvaluator(Tokenizer tokenizer, MissclassifiedSampleListener<TokenSample> sampleListener) {
this.tokenizer = tokenizer;
+ this.sampleListener = sampleListener;
}
/**
@@ -75,12 +80,11 @@ public class TokenizerEvaluator extends
public void evaluateSample(TokenSample reference) {
Span predictions[] = tokenizer.tokenizePos(reference.getText());
- Span[] references = reference.getTokenSpans();
-
- if (isPrintError()) {
- String doc = reference.getText();
- printErrors(references, predictions, reference, new TokenSample(doc,
- predictions), doc);
+ if (this.sampleListener != null) {
+ TokenSample predicted = new TokenSample(reference.getText(), predictions);
+ if(!predicted.equals(reference)) {
+ this.sampleListener.missclassified(reference, predicted);
+ }
}
fmeasure.updateScores(reference.getTokenSpans(), predictions);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/Evaluator.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/Evaluator.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/Evaluator.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/Evaluator.java Wed Aug 17 15:03:28 2011
@@ -19,12 +19,8 @@
package opennlp.tools.util.eval;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.Span;
/**
* The {@link Evaluator} is an abstract base class for evaluators.
@@ -33,22 +29,11 @@ import opennlp.tools.util.Span;
* scores calculated for each reference sample.
*/
public abstract class Evaluator<T> {
-
- private final boolean isPrintError;
-
- public Evaluator(boolean printError) {
- isPrintError = printError;
- }
-
- public Evaluator() {
- isPrintError = false;
- }
/**
* Evaluates the given reference object.
*
- * The implementation has to update the score after every invocation and invoke
- * printErrors(...) if requested by user.
+ * The implementation has to update the score after every invocation.
*
* @param sample the sample to be evaluated
*/
@@ -68,246 +53,4 @@ public abstract class Evaluator<T> {
evaluateSample(sample);
}
}
-
- /**
- * Extensions of this class should check this property to check if should call printErrors method.
- * @return true if to call printErrors method.
- */
- protected final boolean isPrintError() {
- return isPrintError;
- }
-
- /**
- * Prints a report informing errors found in this sample
- *
- * This method should be called by implementations of
- * {@link #evaluateSample(Object)}
- *
- * @param references
- * the reference Span
- * @param predictions
- * the predicted Span
- * @param referenceSample
- * the reference sample
- * @param predictedSample
- * the predicted sample
- * @param doc
- * the document
- */
- protected void printErrors(Span references[], Span predictions[],
- T referenceSample, T predictedSample, String doc) {
-
- List<Span> falseNegatives = new ArrayList<Span>();
- List<Span> falsePositives = new ArrayList<Span>();
-
- findErrors(references, predictions, falseNegatives, falsePositives);
-
- if (falsePositives.size() + falseNegatives.size() > 0) {
-
- printSamples(referenceSample, predictedSample);
-
- printErrors(falsePositives, falseNegatives, doc);
-
- }
- }
-
- /**
- * Prints a report informing errors found in this sample
- *
- * This method should be called by implementations of
- * {@link #evaluateSample(Object)}
- *
- * @param references
- * the reference Span
- * @param predictions
- * the predicted Span
- * @param referenceSample
- * the reference sample
- * @param predictedSample
- * the predicted sample
- * @param doc
- * the document
- */
- protected void printErrors(Span references[], Span predictions[],
- T referenceSample, T predictedSample, String[] doc) {
-
- List<Span> falseNegatives = new ArrayList<Span>();
- List<Span> falsePositives = new ArrayList<Span>();
-
- findErrors(references, predictions, falseNegatives, falsePositives);
-
- if (falsePositives.size() + falseNegatives.size() > 0) {
-
- printSamples(referenceSample, predictedSample);
-
- printErrors(falsePositives, falseNegatives, doc);
-
- }
-
- }
-
- /**
- * Prints a report informing errors found in this sample
- *
- * This method should be called by implementations of
- * {@link #evaluateSample(Object)}
- *
- * @param references
- * the reference tags
- * @param predictions
- * the predicted tags
- * @param referenceSample
- * the reference sample
- * @param predictedSample
- * the predicted sample
- * @param doc
- * the document
- */
- protected void printErrors(String references[], String predictions[],
- T referenceSample, T predictedSample, String[] doc) {
-
- List<String> filteredDoc = new ArrayList<String>();
- List<String> filteredRefs = new ArrayList<String>();
- List<String> filteredPreds = new ArrayList<String>();
-
- for (int i = 0; i < references.length; i++) {
- if (!references[i].equals(predictions[i])) {
- filteredDoc.add(doc[i]);
- filteredRefs.add(references[i]);
- filteredPreds.add(predictions[i]);
- }
- }
-
- if (filteredDoc.size() > 0) {
-
- printSamples(referenceSample, predictedSample);
-
- printErrors(filteredDoc, filteredRefs, filteredPreds);
-
- }
- }
-
- /**
- * Auxiliary method to print tag errors
- *
- * @param filteredDoc
- * the document tokens which were tagged wrong
- * @param filteredRefs
- * the reference tags
- * @param filteredPreds
- * the predicted tags
- */
- private void printErrors(List<String> filteredDoc, List<String> filteredRefs,
- List<String> filteredPreds) {
- System.err.println("Errors: {");
- System.err.println("Tok: Ref | Pred");
- System.err.println("---------------");
- for (int i = 0; i < filteredDoc.size(); i++) {
- System.err.println(filteredDoc.get(i) + ": " + filteredRefs.get(i)
- + " | " + filteredPreds.get(i));
- }
- System.err.println("}\n");
- }
-
- /**
- * Auxiliary method to print span errors
- *
- * @param falsePositives
- * false positives span
- * @param falseNegatives
- * false negative span
- * @param doc
- * the document text
- */
- private void printErrors(List<Span> falsePositives,
- List<Span> falseNegatives, String doc) {
- System.err.println("False positives: {");
- for (Span span : falsePositives) {
- System.err.println(span.getCoveredText(doc));
- }
- System.err.println("} False negatives: {");
- for (Span span : falseNegatives) {
- System.err.println(span.getCoveredText(doc));
- }
- System.err.println("}\n");
- }
-
- /**
- * Auxiliary method to print span errors
- *
- * @param falsePositives
- * false positives span
- * @param falseNegatives
- * false negative span
- * @param toks
- * the document tokens
- */
- private void printErrors(List<Span> falsePositives,
- List<Span> falseNegatives, String[] toks) {
- System.err.println("False positives: {");
- System.err.println(print(falsePositives, toks));
- System.err.println("} False negatives: {");
- System.err.println(print(falseNegatives, toks));
- System.err.println("}\n");
- }
-
- /**
- * Auxiliary method to print spans
- *
- * @param spans
- * the span list
- * @param toks
- * the tokens array
- * @return the spans as string
- */
- private String print(List<Span> spans, String[] toks) {
- return Arrays.toString(Span.spansToStrings(
- spans.toArray(new Span[spans.size()]), toks));
- }
-
- /**
- * Auxiliary method to print expected and predicted samples.
- *
- * @param referenceSample
- * the reference sample
- * @param predictedSample
- * the predicted sample
- */
- private void printSamples(T referenceSample, T predictedSample) {
- String details = "Expected: {\n" + referenceSample + "}\nPredicted: {\n"
- + predictedSample + "}";
- System.err.println(details);
- }
-
- /**
- * Outputs falseNegatives and falsePositives spans from the references and
- * predictions list.
- *
- * @param references
- * @param predictions
- * @param falseNegatives
- * [out] the false negatives list
- * @param falsePositives
- * [out] the false positives list
- */
- private void findErrors(Span references[], Span predictions[],
- List<Span> falseNegatives, List<Span> falsePositives) {
-
- falseNegatives.addAll(Arrays.asList(references));
- falsePositives.addAll(Arrays.asList(predictions));
-
- for (int referenceIndex = 0; referenceIndex < references.length; referenceIndex++) {
-
- Span referenceName = references[referenceIndex];
-
- for (int predictedIndex = 0; predictedIndex < predictions.length; predictedIndex++) {
- if (referenceName.equals(predictions[predictedIndex])) {
- // got it, remove from fn and fp
- falseNegatives.remove(referenceName);
- falsePositives.remove(predictions[predictedIndex]);
- }
- }
- }
- }
-
}
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/MissclassifiedSampleListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/MissclassifiedSampleListener.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/MissclassifiedSampleListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/MissclassifiedSampleListener.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.eval;
+
+public interface MissclassifiedSampleListener<T> {
+
+ void missclassified(T reference, T prediction);
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/MissclassifiedSampleListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java?rev=1158760&r1=1158759&r2=1158760&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerEvaluatorTest.java Wed Aug 17 15:03:28 2011
@@ -17,14 +17,19 @@
package opennlp.tools.chunker;
+import static junit.framework.Assert.assertNotSame;
import static org.junit.Assert.assertEquals;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.OutputStream;
+import opennlp.tools.cmdline.chunker.ChunkEvaluationErrorListener;
import opennlp.tools.util.PlainTextByLineStream;
import opennlp.tools.util.eval.FMeasure;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
import org.junit.Test;
@@ -60,7 +65,10 @@ public class ChunkerEvaluatorTest {
new PlainTextByLineStream(new InputStreamReader(inExpected)), false);
Chunker dummyChunker = new DummyChunker(predictedSample);
- ChunkerEvaluator evaluator = new ChunkerEvaluator(dummyChunker);
+
+ OutputStream stream = new ByteArrayOutputStream();
+ MissclassifiedSampleListener<ChunkSample> listener = new ChunkEvaluationErrorListener(stream);
+ ChunkerEvaluator evaluator = new ChunkerEvaluator(dummyChunker, listener);
evaluator.evaluate(expectedSample);
@@ -68,6 +76,44 @@ public class ChunkerEvaluatorTest {
assertEquals(0.8d, fm.getPrecisionScore(), DELTA);
assertEquals(0.875d, fm.getRecallScore(), DELTA);
+
+ assertNotSame(stream.toString().length(), 0);
+
}
+
+ @Test
+ public void testEvaluatorNoError() throws IOException {
+ InputStream inPredicted = getClass().getClassLoader().getResourceAsStream(
+ "opennlp/tools/chunker/output.txt");
+ InputStream inExpected = getClass().getClassLoader().getResourceAsStream(
+ "opennlp/tools/chunker/output.txt");
+
+ String encoding = "UTF-8";
+
+ DummyChunkSampleStream predictedSample = new DummyChunkSampleStream(
+ new PlainTextByLineStream(new InputStreamReader(inPredicted, encoding)),
+ true);
+
+ DummyChunkSampleStream expectedSample = new DummyChunkSampleStream(
+ new PlainTextByLineStream(new InputStreamReader(inExpected, encoding)),
+ true);
+
+ Chunker dummyChunker = new DummyChunker(predictedSample);
+
+ OutputStream stream = new ByteArrayOutputStream();
+ MissclassifiedSampleListener<ChunkSample> listener = new ChunkEvaluationErrorListener(
+ stream);
+ ChunkerEvaluator evaluator = new ChunkerEvaluator(dummyChunker, listener);
+
+ evaluator.evaluate(expectedSample);
+
+ FMeasure fm = evaluator.getFMeasure();
+
+ assertEquals(1d, fm.getPrecisionScore(), DELTA);
+ assertEquals(1d, fm.getRecallScore(), DELTA);
+
+ assertEquals(stream.toString().length(), 0);
+
+ }
}
Added: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java?rev=1158760&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java Wed Aug 17 15:03:28 2011
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.namefind;
+
+import static junit.framework.Assert.*;
+
+import java.io.ByteArrayOutputStream;
+import java.io.OutputStream;
+
+import opennlp.tools.cmdline.namefind.NameEvaluationErrorListener;
+import opennlp.tools.util.Span;
+import opennlp.tools.util.eval.MissclassifiedSampleListener;
+
+import org.junit.Test;
+
+/**
+ * This is the test class for {@link TokenNameFinderEvaluator}..
+ */
+public class TokenNameFinderEvaluatorTest {
+
+
+ @Test
+ public void testPositive() {
+ OutputStream stream = new ByteArrayOutputStream();
+ MissclassifiedSampleListener<NameSample> listener = new NameEvaluationErrorListener(stream);
+
+ Span[] pred = createSimpleNameSampleA().getNames();
+ TokenNameFinderEvaluator eval = new TokenNameFinderEvaluator(new DummyNameFinder(pred), listener);
+ eval.evaluateSample(createSimpleNameSampleA());
+
+ assertEquals(1.0, eval.getFMeasure().getFMeasure());
+
+ assertEquals(0, stream.toString().length());
+ }
+
+ @Test
+ public void testNegative() {
+ OutputStream stream = new ByteArrayOutputStream();
+ MissclassifiedSampleListener<NameSample> listener = new NameEvaluationErrorListener(stream);
+
+ Span[] pred = createSimpleNameSampleB().getNames();
+ TokenNameFinderEvaluator eval = new TokenNameFinderEvaluator(new DummyNameFinder(pred), listener);
+ eval.evaluateSample(createSimpleNameSampleA());
+
+ assertEquals(0.8, eval.getFMeasure().getFMeasure());
+
+ assertNotSame(0, stream.toString().length());
+ }
+
+
+
+ private static String[] sentence = {"U", ".", "S", ".", "President", "Barack", "Obama", "is",
+ "considering", "sending", "additional", "American", "forces",
+ "to", "Afghanistan", "."};
+
+ private static NameSample createSimpleNameSampleA() {
+
+ Span[] names = { new Span(0, 4, "Location"), new Span(5, 7, "Person"),
+ new Span(14, 15, "Location") };
+
+ NameSample nameSample;
+ nameSample = new NameSample(sentence, names, false);
+
+ return nameSample;
+ }
+
+ private static NameSample createSimpleNameSampleB() {
+
+ Span[] names = { new Span(0, 4, "Location"), new Span(14, 15, "Location") };
+
+ NameSample nameSample;
+ nameSample = new NameSample(sentence, names, false);
+
+ return nameSample;
+ }
+
+ /** a dummy name finder that always return something expected */
+ class DummyNameFinder implements TokenNameFinder {
+
+ private Span[] ret;
+
+ public DummyNameFinder(Span[] ret) {
+ this.ret = ret;
+ }
+
+ public Span[] find(String[] tokens) {
+ return ret;
+ }
+
+ public void clearAdaptiveData() {
+ }
+
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/namefind/TokenNameFinderEvaluatorTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain