You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2011/01/05 17:44:59 UTC
svn commit: r1055519 - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/util/eval/FMeasure.java
test/java/opennlp/tools/util/eval/FMeasureTest.java
Author: colen
Date: Wed Jan 5 16:44:58 2011
New Revision: 1055519
URL: http://svn.apache.org/viewvc?rev=1055519&view=rev
Log:
OPENNLP-59 New strategy to compute Precision, Recall and FM
Modified:
incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java?rev=1055519&r1=1055518&r2=1055519&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java Wed Jan 5 16:44:58 2011
@@ -29,15 +29,13 @@ package opennlp.tools.util.eval;
*/
public final class FMeasure {
- /**
- * The mean of all calculated precision scores.
- */
- private Mean precisionScore = new Mean();
-
- /**
- * The mean of all calculated recall scores.
- */
- private Mean recallScore = new Mean();
+ /** |selected| = tp + fp */
+ private long selected;
+
+ /** |target| = tp + fp */
+ private long target;
+
+ private long truePositive;
/**
* Retrieves the arithmetic mean of the precision scores
@@ -46,7 +44,7 @@ public final class FMeasure {
* @return the arithmetic mean of all precision scores
*/
public double getPrecisionScore() {
- return precisionScore.mean();
+ return selected > 0 ? (double)truePositive / (double)selected : 0;
}
/**
@@ -56,7 +54,7 @@ public final class FMeasure {
* @return the arithmetic mean of all recall scores
*/
public double getRecallScore() {
- return recallScore.mean();
+ return target > 0 ? (double)truePositive / (double)target : 0;
}
/**
@@ -79,19 +77,16 @@ public final class FMeasure {
}
public void updateScores(Object references[], Object predictions[]) {
-
- double precision = FMeasure.precision(references, predictions);
- if (!Double.isNaN(precision))
- precisionScore.add(precision, references.length);
-
- double recall = FMeasure.recall(references, predictions);
- if (!Double.isNaN(recall))
- recallScore.add(FMeasure.recall(references, predictions), references.length);
+
+ truePositive += countTruePositives(references, predictions);
+ selected += predictions.length;
+ target += references.length;
}
public void mergeInto(FMeasure measure) {
- precisionScore.add(measure.getPrecisionScore(), measure.precisionScore.count());
- recallScore.add(measure.getRecallScore(), measure.recallScore.count());
+ this.selected += measure.selected;
+ this.target += measure.target;
+ this.truePositive += measure.truePositive;
}
/**
Modified: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java?rev=1055519&r1=1055518&r2=1055519&view=diff
==============================================================================
--- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java (original)
+++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java Wed Jan 5 16:44:58 2011
@@ -53,6 +53,27 @@ public class FMeasureTest {
new Span(212, 220),
new Span(220, 230)
};
+
+ private Span goldToMerge[] = {
+ new Span(8, 9),
+ new Span(9, 10),
+ new Span(11, 11),
+ new Span(13, 14),
+ new Span(14, 15),
+ new Span(15, 16),
+ new Span(18, 19),
+ };
+
+ private Span predictedToMerge[] = {
+ new Span(8, 9),
+ new Span(14, 15),
+ new Span(15, 16),
+ new Span(100, 120),
+ new Span(210, 220),
+ new Span(220, 230)
+ };
+
+
/**
* Test for the {@link EvaluatorUtil#countTruePositives(Span[], Span[])} method.
@@ -88,4 +109,49 @@ public class FMeasureTest {
assertEquals(Double.NaN, FMeasure.recall(new Object[]{}, gold), DELTA);
assertEquals(2d / gold.length, FMeasure.recall(gold, predicted), DELTA);
}
+
+ @Test
+ public void testEmpty() {
+ FMeasure fm = new FMeasure();
+ assertEquals(-1, fm.getFMeasure(), DELTA);
+ assertEquals(0, fm.getRecallScore(), DELTA);
+ assertEquals(0, fm.getPrecisionScore(), DELTA);
+ }
+
+ @Test
+ public void testPerfect() {
+ FMeasure fm = new FMeasure();
+ fm.updateScores(gold, gold);
+ assertEquals(1, fm.getFMeasure(), DELTA);
+ assertEquals(1, fm.getRecallScore(), DELTA);
+ assertEquals(1, fm.getPrecisionScore(), DELTA);
+ }
+
+ @Test
+ public void testMerge() {
+ FMeasure fm = new FMeasure();
+ fm.updateScores(gold, predicted);
+ fm.updateScores(goldToMerge, predictedToMerge);
+
+ FMeasure fmMerge = new FMeasure();
+ fmMerge.updateScores(gold, predicted);
+ FMeasure toMerge = new FMeasure();
+ toMerge.updateScores(goldToMerge, predictedToMerge);
+ fmMerge.mergeInto(toMerge);
+
+ double selected1 = predicted.length;
+ double target1 = gold.length;
+ double tp1 = FMeasure.countTruePositives(gold, predicted);
+
+ double selected2 = predictedToMerge.length;
+ double target2 = goldToMerge.length;
+ double tp2 = FMeasure.countTruePositives(goldToMerge, predictedToMerge);
+
+
+ assertEquals((tp1 + tp2) / (target1 + target2), fm.getRecallScore(), DELTA);
+ assertEquals((tp1 + tp2) / (selected1 + selected2), fm.getPrecisionScore(), DELTA);
+
+ assertEquals(fm.getRecallScore(), fmMerge.getRecallScore(), DELTA);
+ assertEquals(fm.getPrecisionScore(), fmMerge.getPrecisionScore(), DELTA);
+ }
}
\ No newline at end of file
Re: svn commit: r1055519 - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/util/eval/FMeasure.java test/java/opennlp/tools/util/eval/FMeasureTest.java
Posted by "william.colen@gmail.com" <wi...@gmail.com>.
Thank you, Jörn.
I committed a new revision.
William
On Wed, Jan 12, 2011 at 8:24 AM, Jörn Kottmann <ko...@gmail.com> wrote:
> Hi William,
>
> reviewed your changes, looks nice.
>
> I do not understand the javadoc comment for the fields
> target and selected. Can you improve them ?
>
> Jörn
>
> On 1/5/11 5:44 PM, colen@apache.org wrote:
>
>> Author: colen
>> Date: Wed Jan 5 16:44:58 2011
>> New Revision: 1055519
>>
>> URL: http://svn.apache.org/viewvc?rev=1055519&view=rev
>> Log:
>> OPENNLP-59 New strategy to compute Precision, Recall and FM
>>
>> Modified:
>>
>> incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
>>
>> incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
>>
>> Modified:
>> incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java?rev=1055519&r1=1055518&r2=1055519&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
>> (original)
>> +++
>> incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
>> Wed Jan 5 16:44:58 2011
>> @@ -29,15 +29,13 @@ package opennlp.tools.util.eval;
>> */
>> public final class FMeasure {
>>
>> - /**
>> - * The mean of all calculated precision scores.
>> - */
>> - private Mean precisionScore = new Mean();
>> -
>> - /**
>> - * The mean of all calculated recall scores.
>> - */
>> - private Mean recallScore = new Mean();
>> + /** |selected| = tp + fp */
>> + private long selected;
>> +
>> + /** |target| = tp + fp */
>> + private long target;
>> +
>> + private long truePositive;
>>
>> /**
>> * Retrieves the arithmetic mean of the precision scores
>> @@ -46,7 +44,7 @@ public final class FMeasure {
>> * @return the arithmetic mean of all precision scores
>> */
>> public double getPrecisionScore() {
>> - return precisionScore.mean();
>> + return selected> 0 ? (double)truePositive / (double)selected : 0;
>> }
>>
>> /**
>> @@ -56,7 +54,7 @@ public final class FMeasure {
>> * @return the arithmetic mean of all recall scores
>> */
>> public double getRecallScore() {
>> - return recallScore.mean();
>> + return target> 0 ? (double)truePositive / (double)target : 0;
>> }
>>
>> /**
>> @@ -79,19 +77,16 @@ public final class FMeasure {
>> }
>>
>> public void updateScores(Object references[], Object predictions[]) {
>> -
>> - double precision = FMeasure.precision(references, predictions);
>> - if (!Double.isNaN(precision))
>> - precisionScore.add(precision, references.length);
>> -
>> - double recall = FMeasure.recall(references, predictions);
>> - if (!Double.isNaN(recall))
>> - recallScore.add(FMeasure.recall(references, predictions),
>> references.length);
>> +
>> + truePositive += countTruePositives(references, predictions);
>> + selected += predictions.length;
>> + target += references.length;
>> }
>>
>> public void mergeInto(FMeasure measure) {
>> - precisionScore.add(measure.getPrecisionScore(),
>> measure.precisionScore.count());
>> - recallScore.add(measure.getRecallScore(),
>> measure.recallScore.count());
>> + this.selected += measure.selected;
>> + this.target += measure.target;
>> + this.truePositive += measure.truePositive;
>> }
>>
>> /**
>>
>> Modified:
>> incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
>> URL:
>> http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java?rev=1055519&r1=1055518&r2=1055519&view=diff
>>
>> ==============================================================================
>> ---
>> incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
>> (original)
>> +++
>> incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
>> Wed Jan 5 16:44:58 2011
>> @@ -53,6 +53,27 @@ public class FMeasureTest {
>> new Span(212, 220),
>> new Span(220, 230)
>> };
>> +
>> + private Span goldToMerge[] = {
>> + new Span(8, 9),
>> + new Span(9, 10),
>> + new Span(11, 11),
>> + new Span(13, 14),
>> + new Span(14, 15),
>> + new Span(15, 16),
>> + new Span(18, 19),
>> + };
>> +
>> + private Span predictedToMerge[] = {
>> + new Span(8, 9),
>> + new Span(14, 15),
>> + new Span(15, 16),
>> + new Span(100, 120),
>> + new Span(210, 220),
>> + new Span(220, 230)
>> + };
>> +
>> +
>>
>> /**
>> * Test for the {@link EvaluatorUtil#countTruePositives(Span[],
>> Span[])} method.
>> @@ -88,4 +109,49 @@ public class FMeasureTest {
>> assertEquals(Double.NaN, FMeasure.recall(new Object[]{}, gold),
>> DELTA);
>> assertEquals(2d / gold.length, FMeasure.recall(gold, predicted),
>> DELTA);
>> }
>> +
>> + @Test
>> + public void testEmpty() {
>> + FMeasure fm = new FMeasure();
>> + assertEquals(-1, fm.getFMeasure(), DELTA);
>> + assertEquals(0, fm.getRecallScore(), DELTA);
>> + assertEquals(0, fm.getPrecisionScore(), DELTA);
>> + }
>> +
>> + @Test
>> + public void testPerfect() {
>> + FMeasure fm = new FMeasure();
>> + fm.updateScores(gold, gold);
>> + assertEquals(1, fm.getFMeasure(), DELTA);
>> + assertEquals(1, fm.getRecallScore(), DELTA);
>> + assertEquals(1, fm.getPrecisionScore(), DELTA);
>> + }
>> +
>> + @Test
>> + public void testMerge() {
>> + FMeasure fm = new FMeasure();
>> + fm.updateScores(gold, predicted);
>> + fm.updateScores(goldToMerge, predictedToMerge);
>> +
>> + FMeasure fmMerge = new FMeasure();
>> + fmMerge.updateScores(gold, predicted);
>> + FMeasure toMerge = new FMeasure();
>> + toMerge.updateScores(goldToMerge, predictedToMerge);
>> + fmMerge.mergeInto(toMerge);
>> +
>> + double selected1 = predicted.length;
>> + double target1 = gold.length;
>> + double tp1 = FMeasure.countTruePositives(gold, predicted);
>> +
>> + double selected2 = predictedToMerge.length;
>> + double target2 = goldToMerge.length;
>> + double tp2 = FMeasure.countTruePositives(goldToMerge,
>> predictedToMerge);
>> +
>> +
>> + assertEquals((tp1 + tp2) / (target1 + target2),
>> fm.getRecallScore(), DELTA);
>> + assertEquals((tp1 + tp2) / (selected1 + selected2),
>> fm.getPrecisionScore(), DELTA);
>> +
>> + assertEquals(fm.getRecallScore(), fmMerge.getRecallScore(),
>> DELTA);
>> + assertEquals(fm.getPrecisionScore(),
>> fmMerge.getPrecisionScore(), DELTA);
>> + }
>> }
>> \ No newline at end of file
>>
>>
>>
>
Re: svn commit: r1055519 - in /incubator/opennlp/trunk/opennlp-tools/src:
main/java/opennlp/tools/util/eval/FMeasure.java test/java/opennlp/tools/util/eval/FMeasureTest.java
Posted by Jörn Kottmann <ko...@gmail.com>.
Hi William,
reviewed your changes, looks nice.
I do not understand the javadoc comment for the fields
target and selected. Can you improve them ?
Jörn
On 1/5/11 5:44 PM, colen@apache.org wrote:
> Author: colen
> Date: Wed Jan 5 16:44:58 2011
> New Revision: 1055519
>
> URL: http://svn.apache.org/viewvc?rev=1055519&view=rev
> Log:
> OPENNLP-59 New strategy to compute Precision, Recall and FM
>
> Modified:
> incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
> incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
>
> Modified: incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java
> URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java?rev=1055519&r1=1055518&r2=1055519&view=diff
> ==============================================================================
> --- incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java (original)
> +++ incubator/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/FMeasure.java Wed Jan 5 16:44:58 2011
> @@ -29,15 +29,13 @@ package opennlp.tools.util.eval;
> */
> public final class FMeasure {
>
> - /**
> - * The mean of all calculated precision scores.
> - */
> - private Mean precisionScore = new Mean();
> -
> - /**
> - * The mean of all calculated recall scores.
> - */
> - private Mean recallScore = new Mean();
> + /** |selected| = tp + fp */
> + private long selected;
> +
> + /** |target| = tp + fp */
> + private long target;
> +
> + private long truePositive;
>
> /**
> * Retrieves the arithmetic mean of the precision scores
> @@ -46,7 +44,7 @@ public final class FMeasure {
> * @return the arithmetic mean of all precision scores
> */
> public double getPrecisionScore() {
> - return precisionScore.mean();
> + return selected> 0 ? (double)truePositive / (double)selected : 0;
> }
>
> /**
> @@ -56,7 +54,7 @@ public final class FMeasure {
> * @return the arithmetic mean of all recall scores
> */
> public double getRecallScore() {
> - return recallScore.mean();
> + return target> 0 ? (double)truePositive / (double)target : 0;
> }
>
> /**
> @@ -79,19 +77,16 @@ public final class FMeasure {
> }
>
> public void updateScores(Object references[], Object predictions[]) {
> -
> - double precision = FMeasure.precision(references, predictions);
> - if (!Double.isNaN(precision))
> - precisionScore.add(precision, references.length);
> -
> - double recall = FMeasure.recall(references, predictions);
> - if (!Double.isNaN(recall))
> - recallScore.add(FMeasure.recall(references, predictions), references.length);
> +
> + truePositive += countTruePositives(references, predictions);
> + selected += predictions.length;
> + target += references.length;
> }
>
> public void mergeInto(FMeasure measure) {
> - precisionScore.add(measure.getPrecisionScore(), measure.precisionScore.count());
> - recallScore.add(measure.getRecallScore(), measure.recallScore.count());
> + this.selected += measure.selected;
> + this.target += measure.target;
> + this.truePositive += measure.truePositive;
> }
>
> /**
>
> Modified: incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java
> URL: http://svn.apache.org/viewvc/incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java?rev=1055519&r1=1055518&r2=1055519&view=diff
> ==============================================================================
> --- incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java (original)
> +++ incubator/opennlp/trunk/opennlp-tools/src/test/java/opennlp/tools/util/eval/FMeasureTest.java Wed Jan 5 16:44:58 2011
> @@ -53,6 +53,27 @@ public class FMeasureTest {
> new Span(212, 220),
> new Span(220, 230)
> };
> +
> + private Span goldToMerge[] = {
> + new Span(8, 9),
> + new Span(9, 10),
> + new Span(11, 11),
> + new Span(13, 14),
> + new Span(14, 15),
> + new Span(15, 16),
> + new Span(18, 19),
> + };
> +
> + private Span predictedToMerge[] = {
> + new Span(8, 9),
> + new Span(14, 15),
> + new Span(15, 16),
> + new Span(100, 120),
> + new Span(210, 220),
> + new Span(220, 230)
> + };
> +
> +
>
> /**
> * Test for the {@link EvaluatorUtil#countTruePositives(Span[], Span[])} method.
> @@ -88,4 +109,49 @@ public class FMeasureTest {
> assertEquals(Double.NaN, FMeasure.recall(new Object[]{}, gold), DELTA);
> assertEquals(2d / gold.length, FMeasure.recall(gold, predicted), DELTA);
> }
> +
> + @Test
> + public void testEmpty() {
> + FMeasure fm = new FMeasure();
> + assertEquals(-1, fm.getFMeasure(), DELTA);
> + assertEquals(0, fm.getRecallScore(), DELTA);
> + assertEquals(0, fm.getPrecisionScore(), DELTA);
> + }
> +
> + @Test
> + public void testPerfect() {
> + FMeasure fm = new FMeasure();
> + fm.updateScores(gold, gold);
> + assertEquals(1, fm.getFMeasure(), DELTA);
> + assertEquals(1, fm.getRecallScore(), DELTA);
> + assertEquals(1, fm.getPrecisionScore(), DELTA);
> + }
> +
> + @Test
> + public void testMerge() {
> + FMeasure fm = new FMeasure();
> + fm.updateScores(gold, predicted);
> + fm.updateScores(goldToMerge, predictedToMerge);
> +
> + FMeasure fmMerge = new FMeasure();
> + fmMerge.updateScores(gold, predicted);
> + FMeasure toMerge = new FMeasure();
> + toMerge.updateScores(goldToMerge, predictedToMerge);
> + fmMerge.mergeInto(toMerge);
> +
> + double selected1 = predicted.length;
> + double target1 = gold.length;
> + double tp1 = FMeasure.countTruePositives(gold, predicted);
> +
> + double selected2 = predictedToMerge.length;
> + double target2 = goldToMerge.length;
> + double tp2 = FMeasure.countTruePositives(goldToMerge, predictedToMerge);
> +
> +
> + assertEquals((tp1 + tp2) / (target1 + target2), fm.getRecallScore(), DELTA);
> + assertEquals((tp1 + tp2) / (selected1 + selected2), fm.getPrecisionScore(), DELTA);
> +
> + assertEquals(fm.getRecallScore(), fmMerge.getRecallScore(), DELTA);
> + assertEquals(fm.getPrecisionScore(), fmMerge.getPrecisionScore(), DELTA);
> + }
> }
> \ No newline at end of file
>
>