You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by jo...@apache.org on 2011/09/27 20:30:19 UTC

svn commit: r1176503 - in /incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder: MultiModelNameFinder.java NameFinderJob.java

Author: joern
Date: Tue Sep 27 18:30:18 2011
New Revision: 1176503

URL: http://svn.apache.org/viewvc?rev=1176503&view=rev
Log:
OPENNLP-235 Enabled verified name restriction again.

Modified:
    incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/MultiModelNameFinder.java
    incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderJob.java

Modified: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/MultiModelNameFinder.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/MultiModelNameFinder.java?rev=1176503&r1=1176502&r2=1176503&view=diff
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/MultiModelNameFinder.java (original)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/MultiModelNameFinder.java Tue Sep 27 18:30:18 2011
@@ -37,10 +37,16 @@ public class MultiModelNameFinder implem
 
   static class RestrictedSequencesValidator extends NameFinderSequenceValidator {
     
+    private String modelType;
+    
     private Map<Integer, String> nameIndex = new HashMap<Integer, String>();
     
     private Set<String> nameOnlyTokens;
     
+    RestrictedSequencesValidator(String modelType) {
+      this.modelType = modelType;
+    }
+    
     // also give it a no-name index
     void setRestriction(Map<Integer, String> nameIndex) {
       this.nameIndex = nameIndex;
@@ -56,16 +62,26 @@ public class MultiModelNameFinder implem
       boolean valid = super.validSequence(i, inputSequence, outcomesSequence, outcome);
       
       if (valid && nameIndex.get(i) != null) {
-        String desiredOutcome = nameIndex.get(i);
-        return outcome.endsWith(desiredOutcome);
+        
+        // TODO: That could be improved!
+        String parts[] = nameIndex.get(i).split("-");
+        String nameModelType = parts[0];
+        String desiredOutcome = parts[1];
+        
+        if (modelType.equals(nameModelType)) {
+          return outcome.endsWith(desiredOutcome);
+        }
+        else {
+          return NameFinderME.OTHER.equals(outcome);
+        }
       }
       
       // if token part of name only token, then 
       // its either start, or cont
-      if (valid && nameOnlyTokens.contains(inputSequence[i])) {
-          return outcome.endsWith(NameFinderME.START) || 
-                  outcome.endsWith(NameFinderME.CONTINUE); 
-      }
+//      if (valid && nameOnlyTokens.contains(inputSequence[i])) {
+//          return outcome.endsWith(NameFinderME.START) || 
+//                  outcome.endsWith(NameFinderME.CONTINUE); 
+//      }
       
       return valid;
     }
@@ -73,16 +89,18 @@ public class MultiModelNameFinder implem
   
   
   private NameFinderME nameFinders[];
-  private String typeNames[]; // renmame to modelTypes
+  private String modelTypes[];
   
-  private RestrictedSequencesValidator sequenceValidator;
+  // TODO: We need one per name finder instance ...
+  private RestrictedSequencesValidator sequenceValidators[];
   
-  MultiModelNameFinder(String modelPathes[], String typeNames[]) {
+  MultiModelNameFinder(String modelPathes[], String modelTypes[]) {
     
-    this.typeNames = typeNames;
+    this.modelTypes = modelTypes;
     
     nameFinders = new NameFinderME[modelPathes.length];
-        
+    sequenceValidators = new RestrictedSequencesValidator[modelPathes.length];
+    
     for (int i = 0; i < modelPathes.length; i++) {
       
       String modelPath = modelPathes[i];
@@ -91,9 +109,8 @@ public class MultiModelNameFinder implem
       
       try {
         TokenNameFinderModel model = new TokenNameFinderModel(modelIn);
-//        sequenceValidator = new RestrictedSequencesValidator();
-//        nameFinders[i] = new NameFinderME(model, null, 5, sequenceValidator);
-        nameFinders[i] = new NameFinderME(model, null, 5);
+        sequenceValidators[i] = new RestrictedSequencesValidator(modelTypes[i]);
+        nameFinders[i] = new NameFinderME(model, null, 5, sequenceValidators[i]);
       } catch (IOException e) {
         e.printStackTrace();
       } finally {
@@ -108,14 +125,19 @@ public class MultiModelNameFinder implem
     
   }
   
-  // Needs to be changed, so different models are supported
+  // Note: Outcome value needs to include the type
   void setRestriction(Map<Integer, String> nameIndex) {
-//    sequenceValidator.setRestriction(nameIndex);
+    for (RestrictedSequencesValidator sequenceValidator :  sequenceValidators) {
+      sequenceValidator.setRestriction(nameIndex);
+    }
   }
 
-  // Needs to be changed, so it can be about different categories ... 
   void setNameOnlyTokens(Set<String> nameOnlyTokens) {
-//    sequenceValidator.setNameOnlyTokens(nameOnlyTokens);
+    
+    // How to do this for different types ?!
+//    for (RestrictedSequencesValidator sequenceValidator :  sequenceValidators) {
+//      sequenceValidator.setNameOnlyTokens(nameOnlyTokens);
+//    }
   }
   
   @Override
@@ -138,7 +160,7 @@ public class MultiModelNameFinder implem
       for (int j = 0; j < detectedNames.length; j++) {
         // TODO: Also add type ...
         names.add(new ConfidenceSpan(detectedNames[j].getStart(), detectedNames[j].getEnd(),
-            confidence[j], typeNames[i]));
+            confidence[j], modelTypes[i]));
       }
     }
     

Modified: incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderJob.java
URL: http://svn.apache.org/viewvc/incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderJob.java?rev=1176503&r1=1176502&r2=1176503&view=diff
==============================================================================
--- incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderJob.java (original)
+++ incubator/opennlp/sandbox/caseditor-opennlp-plugin/src/main/java/org/apache/opennlp/caseditor/namefinder/NameFinderJob.java Tue Sep 27 18:30:18 2011
@@ -96,7 +96,7 @@ public class NameFinderJob extends Job {
     
       nameList = new ArrayList<Entity>();
       
-      // remember name tokens, should be for the entire text,
+      // TODO: Name tokens, should be for the entire text,
       // not just the prev sentences ...
       Set<String> nameTokens = new HashSet<String>();
       
@@ -141,7 +141,12 @@ public class NameFinderJob extends Job {
                 outcome = NameFinderME.CONTINUE;
               }
               
-              verifiedNameTokens.put(i, outcome);
+              
+              // TODO: Overlapping names are dangerous here!
+              
+              // TODO: We could use type information here ... 
+              // as part of the outcome!
+              verifiedNameTokens.put(i, verifiedName.getType() + "-" + outcome);
               
               // TODO: Do not put stop word
               // Only put, if char length is two