You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/08/19 03:11:12 UTC
svn commit: r1159467 - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/cmdline/ main/java/opennlp/tools/cmdline/chunker/
main/java/opennlp/tools/cmdline/namefind/ test/java/opennlp/tools/chunker/
test/resources/opennlp/tools/chun...
Author: colen
Date: Fri Aug 19 01:11:12 2011
New Revision: 1159467
URL: http://svn.apache.org/viewvc?rev=1159467&view=rev
Log:
OPENNLP-256 Added initial code for Detailed FMeasure evaluator listener.
Added:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureEvaluatorParams.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderDetailedFMeasureListener.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java (with props)
incubator/opennlp/trunk/opennlp-tools/src/test/resources/opennlp/tools/chunker/detailedOutput.txt (with props)
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureEvaluatorParams.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureEvaluatorParams.java?rev=1159467&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureEvaluatorParams.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureEvaluatorParams.java Fri Aug 19 01:11:12 2011
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline;
+
+import opennlp.tools.cmdline.ArgumentParser.OptionalParameter;
+import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
+
+
+/**
+ * EvaluatorParams for Chunker.
+ *
+ * Note: Do not use this class, internal use only!
+ */
+public interface DetailedFMeasureEvaluatorParams extends EvaluatorParams {
+
+ @ParameterDescription(valueName = "true|false", description = "if true will print detailed FMeasure results")
+ @OptionalParameter(defaultValue="false")
+ Boolean getDetailedF();
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureEvaluatorParams.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java?rev=1159467&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java Fri Aug 19 01:11:12 2011
@@ -0,0 +1,239 @@
+package opennlp.tools.cmdline;
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import opennlp.tools.util.Span;
+import opennlp.tools.util.eval.EvaluationSampleListener;
+
+/**
+ * This listener will gather detailed information about the sample under evaluation and will
+ * allow detailed FMeasure for each outcome.
+ */
+public abstract class DetailedFMeasureListener<T> implements
+ EvaluationSampleListener<T> {
+
+ private int samples = 0;
+ private Stats generalStats = new Stats();
+ private Map<String, Stats> statsForOutcome = new HashMap<String, Stats>();
+
+ protected abstract Span[] asSpanArray(T sample);
+
+ public void correctlyClassified(T reference, T prediction) {
+ samples++;
+ // add all true positives!
+ Span[] spans = asSpanArray(reference);
+ for (Span span : spans) {
+ addTruePositive(span.getType());
+ }
+ }
+
+ public void missclassified(T reference, T prediction) {
+ samples++;
+ Span[] references = asSpanArray(reference);
+ Span[] predictions = asSpanArray(prediction);
+
+ Set<Span> refSet = new HashSet<Span>(Arrays.asList(references));
+ Set<Span> predSet = new HashSet<Span>(Arrays.asList(predictions));
+
+ for (Span ref : refSet) {
+ if (predSet.contains(ref)) {
+ addTruePositive(ref.getType());
+ } else {
+ addFalseNegative(ref.getType());
+ }
+ }
+
+ for (Span pred : predSet) {
+ if (!refSet.contains(pred)) {
+ addFalsePositive(pred.getType());
+ }
+ }
+ }
+
+ private void addTruePositive(String type) {
+ Stats s = initStatsForOutcomeAndGet(type);
+ s.incrementTruePositive();
+ s.incrementTarget();
+
+ generalStats.incrementTruePositive();
+ generalStats.incrementTarget();
+ }
+
+ private void addFalsePositive(String type) {
+ Stats s = initStatsForOutcomeAndGet(type);
+ s.incrementFalsePositive();
+ generalStats.incrementFalsePositive();
+ }
+
+ private void addFalseNegative(String type) {
+ Stats s = initStatsForOutcomeAndGet(type);
+ s.incrementTarget();
+ generalStats.incrementTarget();
+
+ }
+
+ private Stats initStatsForOutcomeAndGet(String type) {
+ if (!statsForOutcome.containsKey(type)) {
+ statsForOutcome.put(type, new Stats());
+ }
+ return statsForOutcome.get(type);
+ }
+
+ private static final String PERCENT = "%\u00207.2f%%";
+ private static final String FORMAT = "%8s: precision: " + PERCENT
+ + "; recall: " + PERCENT + "; F1: " + PERCENT + ".";
+ private static final String FORMAT_EXTRA = FORMAT
+ + " [target: %3d; tp: %3d; fp: %3d]";
+
+ @Override
+ public String toString() {
+ StringBuilder ret = new StringBuilder();
+ int tp = generalStats.getTruePositives();
+ int found = generalStats.getFalsePositives() + tp;
+ ret.append("processed " + samples + " samples with "
+ + generalStats.getTarget() + " entities; found: " + found
+ + " entities; correct: " + tp + ".\n");
+
+ ret.append(String.format(FORMAT, "TOTAL",
+ zeroOrPositive(generalStats.getPrecisionScore() * 100),
+ zeroOrPositive(generalStats.getRecallScore() * 100),
+ zeroOrPositive(generalStats.getFMeasure() * 100)));
+ ret.append("\n");
+ SortedSet<String> set = new TreeSet<String>(new F1Comparator());
+ set.addAll(statsForOutcome.keySet());
+ for (String type : set) {
+
+ ret.append(String.format(FORMAT_EXTRA, type,
+ zeroOrPositive(statsForOutcome.get(type).getPrecisionScore() * 100),
+ zeroOrPositive(statsForOutcome.get(type).getRecallScore() * 100),
+ zeroOrPositive(statsForOutcome.get(type).getFMeasure() * 100),
+ statsForOutcome.get(type).getTarget(), statsForOutcome.get(type)
+ .getTruePositives(), statsForOutcome.get(type)
+ .getFalsePositives()));
+ ret.append("\n");
+ }
+
+ return ret.toString();
+
+ }
+
+ private double zeroOrPositive(double v) {
+ if (v < 0) {
+ return 0;
+ }
+ return v;
+ }
+
+ private class F1Comparator implements Comparator<String> {
+ public int compare(String o1, String o2) {
+ if (o1.equals(o2))
+ return 0;
+ double t1 = 0;
+ double t2 = 0;
+
+ if (statsForOutcome.containsKey(o1))
+ t1 += statsForOutcome.get(o1).getFMeasure();
+ if (statsForOutcome.containsKey(o2))
+ t2 += statsForOutcome.get(o2).getFMeasure();
+
+ t1 = zeroOrPositive(t1);
+ t2 = zeroOrPositive(t2);
+
+ if (t1 + t2 > 0d) {
+ if (t1 > t2)
+ return -1;
+ return 1;
+ }
+ return o1.compareTo(o2);
+ }
+
+ }
+
+ /**
+ * Store the statistics.
+ */
+ private class Stats {
+
+ // maybe we could use FMeasure class, but it wouldn't allow us to get
+ // details like total number of false positives and true positives.
+
+ private int falsePositiveCounter = 0;
+ private int truePositiveCounter = 0;
+ private int targetCounter = 0;
+
+ public void incrementFalsePositive() {
+ falsePositiveCounter++;
+ }
+
+ public void incrementTruePositive() {
+ truePositiveCounter++;
+ }
+
+ public void incrementTarget() {
+ targetCounter++;
+ }
+
+ public int getFalsePositives() {
+ return falsePositiveCounter;
+ }
+
+ public int getTruePositives() {
+ return truePositiveCounter;
+ }
+
+ public int getTarget() {
+ return targetCounter;
+ }
+
+ /**
+ * Retrieves the arithmetic mean of the precision scores calculated for each
+ * evaluated sample.
+ *
+ * @return the arithmetic mean of all precision scores
+ */
+ public double getPrecisionScore() {
+ int tp = getTruePositives();
+ int selected = tp + getFalsePositives();
+ return selected > 0 ? (double) tp / (double) selected : 0;
+ }
+
+ /**
+ * Retrieves the arithmetic mean of the recall score calculated for each
+ * evaluated sample.
+ *
+ * @return the arithmetic mean of all recall scores
+ */
+ public double getRecallScore() {
+ int target = getTarget();
+ int tp = getTruePositives();
+ return target > 0 ? (double) tp / (double) target : 0;
+ }
+
+ /**
+ * Retrieves the f-measure score.
+ *
+ * f-measure = 2 * precision * recall / (precision + recall)
+ *
+ * @return the f-measure or -1 if precision + recall <= 0
+ */
+ public double getFMeasure() {
+
+ if (getPrecisionScore() + getRecallScore() > 0) {
+ return 2 * (getPrecisionScore() * getRecallScore())
+ / (getPrecisionScore() + getRecallScore());
+ } else {
+ // cannot divide by zero, return error code
+ return -1;
+ }
+ }
+
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/DetailedFMeasureListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java?rev=1159467&r1=1159466&r2=1159467&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerCrossValidatorTool.java Fri Aug 19 01:11:12 2011
@@ -19,7 +19,8 @@ package opennlp.tools.cmdline.chunker;
import java.io.File;
import java.io.IOException;
-import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import opennlp.tools.chunker.ChunkSample;
import opennlp.tools.chunker.ChunkerCrossValidator;
@@ -28,14 +29,15 @@ import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CVParams;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
+import opennlp.tools.cmdline.DetailedFMeasureEvaluatorParams;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.util.ObjectStream;
-import opennlp.tools.util.eval.FMeasure;
import opennlp.tools.util.eval.EvaluationSampleListener;
+import opennlp.tools.util.eval.FMeasure;
public final class ChunkerCrossValidatorTool implements CmdLineTool {
- interface CVToolParams extends TrainingParams, CVParams {
+ interface CVToolParams extends TrainingParams, CVParams, DetailedFMeasureEvaluatorParams {
}
@@ -71,13 +73,16 @@ public final class ChunkerCrossValidator
ChunkerCrossValidator validator = new ChunkerCrossValidator(
params.getLang(), params.getCutoff(), params.getIterations());
- EvaluationSampleListener<ChunkSample> errorListener = null;
- if(params.getMisclassified()) {
- errorListener = new ChunkEvaluationErrorListener();
+ List<EvaluationSampleListener<ChunkSample>> listeners = new LinkedList<EvaluationSampleListener<ChunkSample>>();
+ if (params.getMisclassified()) {
+ listeners.add(new ChunkEvaluationErrorListener());
+ }
+ if (params.getDetailedF()) {
+ listeners.add(new ChunkerDetailedFMeasureListener());
}
try {
- validator.evaluate(sampleStream, params.getFolds(), Collections.singletonList(errorListener));
+ validator.evaluate(sampleStream, params.getFolds(), listeners);
}
catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListener.java?rev=1159467&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListener.java Fri Aug 19 01:11:12 2011
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.chunker;
+
+import opennlp.tools.chunker.ChunkSample;
+import opennlp.tools.cmdline.DetailedFMeasureListener;
+import opennlp.tools.util.Span;
+
+public class ChunkerDetailedFMeasureListener extends
+ DetailedFMeasureListener<ChunkSample> {
+
+ @Override
+ protected Span[] asSpanArray(ChunkSample sample) {
+ return sample.getPhrasesAsSpanList();
+ }
+
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerDetailedFMeasureListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java?rev=1159467&r1=1159466&r2=1159467&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/chunker/ChunkerEvaluatorTool.java Fri Aug 19 01:11:12 2011
@@ -20,23 +20,29 @@ package opennlp.tools.cmdline.chunker;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
-import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import opennlp.tools.chunker.ChunkSample;
import opennlp.tools.chunker.ChunkerEvaluator;
import opennlp.tools.chunker.ChunkerME;
import opennlp.tools.chunker.ChunkerModel;
import opennlp.tools.cmdline.ArgumentParser;
-import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
+import opennlp.tools.cmdline.DetailedFMeasureEvaluatorParams;
+import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.PerformanceMonitor;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.util.ObjectStream;
import opennlp.tools.util.eval.EvaluationSampleListener;
public final class ChunkerEvaluatorTool implements CmdLineTool {
+
+ interface EvalToolParams extends EvaluatorParams, DetailedFMeasureEvaluatorParams {
+
+ }
public String getName() {
return "ChunkerEvaluator";
@@ -57,7 +63,7 @@ public final class ChunkerEvaluatorTool
throw new TerminateToolException(1);
}
- EvaluatorParams params = ArgumentParser.parse(args, EvaluatorParams.class);
+ EvalToolParams params = ArgumentParser.parse(args, EvalToolParams.class);
File testData =params.getData();
@@ -67,13 +73,16 @@ public final class ChunkerEvaluatorTool
ChunkerModel model = new ChunkerModelLoader().load(params.getModel());
- EvaluationSampleListener<ChunkSample> errorListener = null;
+ List<EvaluationSampleListener<ChunkSample>> listeners = new LinkedList<EvaluationSampleListener<ChunkSample>>();
if(params.getMisclassified()) {
- errorListener = new ChunkEvaluationErrorListener();
+ listeners.add(new ChunkEvaluationErrorListener());
+ }
+ if(params.getDetailedF()) {
+ listeners.add(new ChunkerDetailedFMeasureListener());
}
ChunkerEvaluator evaluator = new ChunkerEvaluator(new ChunkerME(model),
- Collections.singletonList(errorListener));
+ listeners);
final ObjectStream<ChunkSample> sampleStream = ChunkerTrainerTool.openSampleData("Test",
testData, encoding);
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java?rev=1159467&r1=1159466&r2=1159467&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderCrossValidatorTool.java Fri Aug 19 01:11:12 2011
@@ -20,7 +20,8 @@ package opennlp.tools.cmdline.namefind;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
-import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import opennlp.tools.cmdline.ArgumentParser;
@@ -28,6 +29,7 @@ import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CVParams;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
+import opennlp.tools.cmdline.DetailedFMeasureEvaluatorParams;
import opennlp.tools.cmdline.TerminateToolException;
import opennlp.tools.namefind.NameSample;
import opennlp.tools.namefind.TokenNameFinderCrossValidator;
@@ -36,7 +38,7 @@ import opennlp.tools.util.eval.Evaluatio
public final class TokenNameFinderCrossValidatorTool implements CmdLineTool {
- interface CVToolParams extends TrainingParams, CVParams{
+ interface CVToolParams extends TrainingParams, CVParams, DetailedFMeasureEvaluatorParams{
}
@@ -80,9 +82,12 @@ public final class TokenNameFinderCrossV
TokenNameFinderCrossValidator validator;
- EvaluationSampleListener<NameSample> errorListener = null;
+ List<EvaluationSampleListener<NameSample>> listeners = new LinkedList<EvaluationSampleListener<NameSample>>();
if (params.getMisclassified()) {
- errorListener = new NameEvaluationErrorListener();
+ listeners.add(new NameEvaluationErrorListener());
+ }
+ if (params.getDetailedF()) {
+ listeners.add(new TokenNameFinderDetailedFMeasureListener());
}
try {
@@ -94,7 +99,7 @@ public final class TokenNameFinderCrossV
validator = new TokenNameFinderCrossValidator(params.getLang(), params.getType(), mlParams,
featureGeneratorBytes, resources);
}
- validator.evaluate(sampleStream, params.getFolds(), Collections.singletonList(errorListener));
+ validator.evaluate(sampleStream, params.getFolds(), listeners);
} catch (IOException e) {
CmdLineUtil.printTrainingIoError(e);
throw new TerminateToolException(-1);
Added: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderDetailedFMeasureListener.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderDetailedFMeasureListener.java?rev=1159467&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderDetailedFMeasureListener.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderDetailedFMeasureListener.java Fri Aug 19 01:11:12 2011
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.namefind;
+
+import opennlp.tools.cmdline.DetailedFMeasureListener;
+import opennlp.tools.namefind.NameSample;
+import opennlp.tools.util.Span;
+
+public class TokenNameFinderDetailedFMeasureListener extends
+ DetailedFMeasureListener<NameSample> {
+
+ @Override
+ protected Span[] asSpanArray(NameSample sample) {
+ return sample.getNames();
+ }
+
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderDetailedFMeasureListener.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java?rev=1159467&r1=1159466&r2=1159467&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderEvaluatorTool.java Fri Aug 19 01:11:12 2011
@@ -20,12 +20,14 @@ package opennlp.tools.cmdline.namefind;
import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
-import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import opennlp.tools.cmdline.ArgumentParser;
import opennlp.tools.cmdline.CLI;
import opennlp.tools.cmdline.CmdLineTool;
import opennlp.tools.cmdline.CmdLineUtil;
+import opennlp.tools.cmdline.DetailedFMeasureEvaluatorParams;
import opennlp.tools.cmdline.EvaluatorParams;
import opennlp.tools.cmdline.PerformanceMonitor;
import opennlp.tools.cmdline.TerminateToolException;
@@ -38,6 +40,10 @@ import opennlp.tools.util.eval.Evaluatio
public final class TokenNameFinderEvaluatorTool implements CmdLineTool {
+ interface EvalToolParams extends EvaluatorParams, DetailedFMeasureEvaluatorParams {
+
+ }
+
public String getName() {
return "TokenNameFinderEvaluator";
}
@@ -59,8 +65,8 @@ public final class TokenNameFinderEvalua
throw new TerminateToolException(1);
}
- EvaluatorParams params = ArgumentParser.parse(args,
- EvaluatorParams.class);
+ EvalToolParams params = ArgumentParser.parse(args,
+ EvalToolParams.class);
File testData = params.getData();
CmdLineUtil.checkInputFile("Test data", testData);
@@ -70,13 +76,16 @@ public final class TokenNameFinderEvalua
TokenNameFinderModel model = new TokenNameFinderModelLoader().load(params
.getModel());
- EvaluationSampleListener<NameSample> missclassifiedListener = null;
+ List<EvaluationSampleListener<NameSample>> listeners = new LinkedList<EvaluationSampleListener<NameSample>>();
if (params.getMisclassified()) {
- missclassifiedListener = new NameEvaluationErrorListener();
+ listeners.add(new NameEvaluationErrorListener());
+ }
+ if (params.getDetailedF()) {
+ listeners.add(new TokenNameFinderDetailedFMeasureListener());
}
TokenNameFinderEvaluator evaluator = new TokenNameFinderEvaluator(
- new NameFinderME(model), Collections.singletonList(missclassifiedListener));
+ new NameFinderME(model), listeners);
final ObjectStream<NameSample> sampleStream = TokenNameFinderTrainerTool.openSampleData("Test",
testData, encoding);
Added: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java?rev=1159467&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java Fri Aug 19 01:11:12 2011
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.chunker;
+
+import static junit.framework.Assert.*;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Collections;
+
+import opennlp.tools.cmdline.chunker.ChunkerDetailedFMeasureListener;
+import opennlp.tools.util.PlainTextByLineStream;
+import opennlp.tools.util.eval.EvaluationSampleListener;
+
+import org.junit.Test;
+
+public class ChunkerDetailedFMeasureListenerTest {
+
+ @Test
+ public void testEvaluator() throws IOException {
+ InputStream inPredicted = getClass().getClassLoader().getResourceAsStream(
+ "opennlp/tools/chunker/output.txt");
+ InputStream inExpected = getClass().getClassLoader().getResourceAsStream(
+ "opennlp/tools/chunker/output.txt");
+
+ InputStream detailedOutputStream = getClass().getClassLoader().getResourceAsStream(
+ "opennlp/tools/chunker/detailedOutput.txt");
+
+ String encoding = "UTF-8";
+
+ try {
+ DummyChunkSampleStream predictedSample = new DummyChunkSampleStream(
+ new PlainTextByLineStream(
+ new InputStreamReader(inPredicted, encoding)), true);
+
+ DummyChunkSampleStream expectedSample = new DummyChunkSampleStream(
+ new PlainTextByLineStream(new InputStreamReader(inExpected)), false);
+
+ Chunker dummyChunker = new DummyChunker(predictedSample);
+
+ EvaluationSampleListener<ChunkSample> listener = new ChunkerDetailedFMeasureListener();
+ ChunkerEvaluator evaluator = new ChunkerEvaluator(dummyChunker,
+ Collections.singletonList(listener));
+
+ evaluator.evaluate(expectedSample);
+
+ StringBuilder expected = new StringBuilder();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(detailedOutputStream, encoding));
+ String line = reader.readLine();
+
+ while(line != null ) {
+ expected.append(line);
+ expected.append("\n");
+ line = reader.readLine();
+ }
+ assertEquals(expected.toString().trim(), listener.toString().trim());
+ } finally {
+ inPredicted.close();
+ inExpected.close();
+ detailedOutputStream.close();
+ }
+ }
+}
Propchange: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/chunker/ChunkerDetailedFMeasureListenerTest.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: incubator/opennlp/trunk/opennlp-tools/src/test/resources/opennlp/tools/chunker/detailedOutput.txt
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/resources/opennlp/tools/chunker/detailedOutput.txt?rev=1159467&view=auto
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/resources/opennlp/tools/chunker/detailedOutput.txt (added)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/resources/opennlp/tools/chunker/detailedOutput.txt Fri Aug 19 01:11:12 2011
@@ -0,0 +1,7 @@
+processed 3 samples with 32 entities; found: 35 entities; correct: 28.
+ TOTAL: precision: 80,00%; recall: 87,50%; F1: 83,58%.
+ NP: precision: 90,00%; recall: 94,74%; F1: 92,31%. [target: 19; tp: 18; fp: 2]
+ VP: precision: 75,00%; recall: 75,00%; F1: 75,00%. [target: 8; tp: 6; fp: 2]
+ PP: precision: 57,14%; recall: 100,00%; F1: 72,73%. [target: 4; tp: 4; fp: 3]
+ SBAR: precision: 0,00%; recall: 0,00%; F1: 0,00%. [target: 1; tp: 0; fp: 0]
+
Propchange: incubator/opennlp/trunk/opennlp-tools/src/test/resources/opennlp/tools/chunker/detailedOutput.txt
------------------------------------------------------------------------------
svn:mime-type = text/plain