svn commit: r1334959 - /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java

[jo...@apache.org]

Author: joern
Date: Mon May  7 11:28:34 2012
New Revision: 1334959

URL: http://svn.apache.org/viewvc?rev=1334959&view=rev
Log:
OPENNLP-342 Fixed one more bug in compound word handling.

Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java?rev=1334959&r1=1334958&r2=1334959&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/formats/frenchtreebank/ConstitDocumentHandler.java Mon May  7 11:28:34 2012
@@ -34,7 +34,6 @@ class ConstitDocumentHandler extends Def
   
   private static final String SENT_ELEMENT_NAME = "SENT";
   private static final String WORD_ELEMENT_NAME = "w";
-  private static final String COMPOUND_ATTR_NAME = "compound";
   
   private static final String SENT_TYPE_NAME = "S";
   
@@ -58,7 +57,8 @@ class ConstitDocumentHandler extends Def
     this.parses = parses;
   }
   
-  private String compoundCat;
+  private String cat;
+  private String subcat;
   
   @Override
   public void startElement(String uri, String localName, String qName,
@@ -66,8 +66,6 @@ class ConstitDocumentHandler extends Def
     
     String type = qName;
 
-    boolean isCompoundWord = false;
-    
     if (SENT_ELEMENT_NAME.equals(qName)) {
       // Clear everything to be ready for the next sentence
       text.setLength(0);
@@ -81,24 +79,43 @@ class ConstitDocumentHandler extends Def
     }
     else if (WORD_ELEMENT_NAME.equals(qName)) {
       
-      // insideCompoundElement
-      if (attributes.getValue(COMPOUND_ATTR_NAME) != null) {
-        isCompoundWord = "yes".equals(attributes.getValue(COMPOUND_ATTR_NAME));
-      }
-      
-      String cat = attributes.getValue("cat");
-      
-      if (isCompoundWord) {
-        compoundCat = cat;
+      // Note:
+      // If there are compound words they are represented in a couple
+      // of ways in the training data.
+      // Many of them are marked with the compound attribute, but not 
+      // all of them. Thats why it is not used in the code to detect
+      // a compound word.
+      // Compounds are detected by the fact that a w tag is appearing
+      // inside a w tag.
+      //
+      // The type of a compound word can be encoded either cat of the compound
+      // plus the catint of each word of the compound.
+      // Or all compound words have the cat plus subcat of the compound, in this
+      // case they have an empty cat attribute.
+      //
+      // This implementation hopefully decodes these cases correctly!
+      
+      String newCat = attributes.getValue("cat");
+      if (newCat != null && newCat.length() > 0) {
+        cat = newCat;
+      }
+      
+      String newSubcat = attributes.getValue("subcat");
+      if (newSubcat != null && newSubcat.length() > 0) {
+        subcat = newSubcat;
       }
       
       if (cat != null) {
-        String subcat = attributes.getValue("subcat");
         type = cat + (subcat != null ? subcat : "");
       }
       else {
         String catint = attributes.getValue("catint");
-        type = compoundCat + (catint != null ? catint : "");
+        if (catint != null) {
+          type = cat + (catint != null ? catint : "");
+        }
+        else {
+          type = cat + subcat;
+        }
       }
     }
     
@@ -139,7 +156,7 @@ class ConstitDocumentHandler extends Def
       if (isCreateConstituent) {
         int start = unfinishedCon.getSpan().getStart();
         if (start < offset) {
-          cons.add(new Constituent(unfinishedCon.getLabel(), new Span(start, offset-1)));
+          cons.add(new Constituent(unfinishedCon.getLabel(), new Span(start, offset - 1)));
         }
       }