svn commit: r1593530 - in /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools: parser/ParserEvaluator.java util/eval/ParseEval.java

[ra...@apache.org]

Author: ragerri
Date: Fri May  9 12:34:08 2014
New Revision: 1593530

URL: http://svn.apache.org/r1593530
Log:
OPENNLP-31 add evaluation suppor to parser working

Added:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/ParseEval.java
Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java?rev=1593530&r1=1593529&r2=1593530&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/parser/ParserEvaluator.java Fri May  9 12:34:08 2014
@@ -23,28 +23,49 @@ import java.util.Stack;
 
 import opennlp.tools.cmdline.parser.ParserTool;
 import opennlp.tools.util.Span;
+import opennlp.tools.util.eval.ParseEval;
 import opennlp.tools.util.eval.Evaluator;
-import opennlp.tools.util.eval.FMeasure;
 
+/**
+ * Class for Parsing Evaluation. Hopefully to be merged
+ * into FMeasure soon.
+ *
+ */
 public class ParserEvaluator extends Evaluator<Parse> {
 
-  private FMeasure fmeasure = new FMeasure();
-
+  /**
+   * fmeasure.
+   */
+  private ParseEval fmeasure = new ParseEval();
+  /**
+   * The parser to evaluate.
+   */
   private final Parser parser;
 
-  public ParserEvaluator(Parser parser, ParserEvaluationMonitor... monitors) {
+  /**
+   * Construct a parser with some evaluation monitors.
+   * @param aParser
+   * @param monitors the evaluation monitors
+   */
+  public ParserEvaluator(final Parser aParser, final ParserEvaluationMonitor... monitors) {
     super(monitors);
-    this.parser = parser;
+    this.parser = aParser;
   }
 
-  private static Span[] getConstituencySpans(Parse parse) {
+  /**
+   * Obtain {@code Span}s for every parse in the sentence.
+   * @param parse
+   * @return an array containing every span for the parse
+   */
+  private static Span[] getConstituencySpans(final Parse parse) {
 
     Stack<Parse> stack = new Stack<Parse>();
 
     if (parse.getChildCount() > 0) {
-      stack.add(parse.getChildren()[0]);
+      for (Parse child : parse.getChildren()) {
+        stack.push(child);
+      }
     }
-
     List<Span> consts = new ArrayList<Span>();
 
     while (!stack.isEmpty()) {
@@ -65,11 +86,11 @@ public class ParserEvaluator extends Eva
   }
 
   @Override
-  protected Parse processSample(Parse reference) {
+  protected final Parse processSample(final Parse reference) {
 
     String sentenceText = reference.getText();
 
-    Parse predictions[] = ParserTool.parseLine(sentenceText, parser, 1);
+    Parse[] predictions = ParserTool.parseLine(sentenceText, parser, 1);
 
     Parse prediction = null;
     if (predictions.length > 0) {
@@ -81,21 +102,29 @@ public class ParserEvaluator extends Eva
     return prediction;
   }
 
-  public FMeasure getFMeasure() {
+  /**
+   * It returns the fmeasure result.
+   * @return the fmeasure value
+   */
+  public final ParseEval getFMeasure() {
     return fmeasure;
   }
 
-  public static void main(String[] args) {
-
-    // TODO: Move this to a test case!
+  /**
+   * Main method to show the example of running the evaluator.
+   * Moved to a test case soon, hopefully.
+   * @param args
+   */
+  // TODO: Move this to a test case!
+  public static void main(final String[] args) {
 
     String goldParseString = "(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care))) ))  (NP (NN yesterday)) (. .) ))";
-    Span goldConsts[] = getConstituencySpans(Parse.parseParse(goldParseString));
+    Span[] goldConsts = getConstituencySpans(Parse.parseParse(goldParseString));
 
     String testParseString = "(TOP (S (NP (NNS Sales) (NNS executives)) (VP (VBD were) (VP (VBG examing) (NP (DT the) (NNS figures)) (PP (IN with) (NP (JJ great) (NN care) (NN yesterday))) ))  (. .) ))";
-    Span testConsts[] = getConstituencySpans(Parse.parseParse(testParseString));
+    Span[] testConsts = getConstituencySpans(Parse.parseParse(testParseString));
 
-    FMeasure measure = new FMeasure();
+    ParseEval measure = new ParseEval();
     measure.updateScores(goldConsts, testConsts);
 
     // Expected output:

Added: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/ParseEval.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/ParseEval.java?rev=1593530&view=auto
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/ParseEval.java (added)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/util/eval/ParseEval.java Fri May  9 12:34:08 2014
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.util.eval;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * The {@link ParseEval} is an utility class for evaluators which measure<br>
+ * precision, recall and the resulting f-measure.
+ *
+ * Evaluation results are the arithmetic mean of the precision scores calculated
+ * for each reference sample and the arithmetic mean of the recall scores
+ * calculated for each reference sample.
+ */
+
+public final class ParseEval {
+
+  /**
+   * |selected| = true positives + false positives <br>
+   * the count of selected (or retrieved) items.
+   */
+  private long selected;
+
+  /**
+   * |target| = true positives + false negatives <br>
+   * the count of target (or correct) items.
+   */
+  private long target;
+
+  /**
+   * Storing the number of true positives found.
+   */
+  private long truePositive;
+
+  /**
+   * Retrieves the arithmetic mean of the precision scores calculated for each
+   * evaluated sample.
+   *
+   * @return the arithmetic mean of all precision scores
+   */
+  public double getPrecisionScore() {
+    return selected > 0 ? (double) truePositive / (double) selected : 0;
+  }
+
+  /**
+   * Retrieves the arithmetic mean of the recall score calculated for each
+   * evaluated sample.
+   *
+   * @return the arithmetic mean of all recall scores
+   */
+  public double getRecallScore() {
+    return target > 0 ? (double) truePositive / (double) target : 0;
+  }
+
+  /**
+   * Retrieves the f-measure score.
+   *
+   * f-measure = 2 * precision * recall / (precision + recall)
+   * @return the f-measure or -1 if precision + recall &lt;= 0
+   */
+  public double getFMeasure() {
+
+    if (getPrecisionScore() + getRecallScore() > 0) {
+      return 2 * (getPrecisionScore() * getRecallScore())
+          / (getPrecisionScore() + getRecallScore());
+    } else {
+      // cannot divide by zero, return error code
+      return -1;
+    }
+  }
+
+  /**
+   * Updates the score based on the number of true positives and
+   * the number of predictions and references.
+   *
+   * @param references the provided references
+   * @param predictions the predicted spans
+   */
+  public void updateScores(final Object[] references, final Object[] predictions) {
+
+    truePositive += countTruePositivesParse(references, predictions);
+    selected += predictions.length;
+    target += references.length;
+  }
+
+  /**
+   * Merge results into fmeasure metric.
+   * @param measure the fmeasure
+   */
+  public void mergeInto(final ParseEval measure) {
+    this.selected += measure.selected;
+    this.target += measure.target;
+    this.truePositive += measure.truePositive;
+  }
+
+  /**
+   * Creates a human read-able {@link String} representation.
+   * @return the results
+   */
+  @Override
+  public String toString() {
+    return "Precision: " + Double.toString(getPrecisionScore()) + "\n"
+        + "Recall: " + Double.toString(getRecallScore()) + "\n" + "F-Measure: "
+        + Double.toString(getFMeasure());
+  }
+
+  /**
+   * This method counts the number of objects which are equal and occur in the
+   * references and predictions arrays.
+   * These are the number of true positives.
+   *
+   * @param references
+   *          the gold standard
+   * @param predictions
+   *          the predictions
+   * @return number of true positives
+   */
+  static int countTruePositivesParse(final Object[] references, final Object[] predictions) {
+
+    List<Object> predListSpans = new ArrayList<Object>(predictions.length);
+    Collections.addAll(predListSpans, predictions);
+    int truePositives = 0;
+    Object matchedItem = null;
+
+    for (int referenceIndex = 0; referenceIndex < references.length; referenceIndex++) {
+      Object referenceName = references[referenceIndex];
+
+      for (int predIndex = 0; predIndex < predListSpans.size(); predIndex++) {
+
+        if (referenceName.equals(predListSpans.get(predIndex))) {
+          matchedItem = predListSpans.get(predIndex);
+          truePositives++;
+        }
+      }
+      if (matchedItem != null) {
+        predListSpans.remove(matchedItem);
+      }
+    }
+    return truePositives;
+  }
+
+
+  /**
+   * Calculates the precision score for the given reference and predicted spans.
+   *
+   * @param references
+   *          the gold standard spans
+   * @param predictions
+   *          the predicted spans
+   * @return the precision score or NaN if there are no predicted spans
+   */
+  public static double precision(final Object[] references, final Object[] predictions) {
+
+    if (predictions.length > 0) {
+      return countTruePositivesParse(references, predictions)
+          / (double) predictions.length;
+    } else {
+      return Double.NaN;
+    }
+  }
+
+  /**
+   * Calculates the recall score for the given reference and predicted spans.
+   *
+   * @param references
+   *          the gold standard spans
+   * @param predictions
+   *          the predicted spans
+   *
+   * @return the recall score or NaN if there are no reference spans
+   */
+  public static double recall(final Object[] references, final Object[] predictions) {
+
+    if (references.length > 0) {
+      return countTruePositivesParse(references, predictions)
+          / (double) references.length;
+    } else {
+      return Double.NaN;
+    }
+  }
+}