svn commit: r1309371 - in /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools: cmdline/namefind/TokenNameFinderTrainerTool.java namefind/NameSampleTypeFilter.java

[jo...@apache.org]

Author: joern
Date: Wed Apr  4 13:25:03 2012
New Revision: 1309371

URL: http://svn.apache.org/viewvc?rev=1309371&view=rev
Log:
OPENNLP-493 Added option to only use selected name types for training.

Added:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java   (with props)
Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java?rev=1309371&r1=1309370&r2=1309371&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/cmdline/namefind/TokenNameFinderTrainerTool.java Wed Apr  4 13:25:03 2012
@@ -26,9 +26,11 @@ import java.util.Map;
 import opennlp.tools.cmdline.AbstractTrainerTool;
 import opennlp.tools.cmdline.CmdLineUtil;
 import opennlp.tools.cmdline.TerminateToolException;
+import opennlp.tools.cmdline.ArgumentParser.ParameterDescription;
 import opennlp.tools.cmdline.namefind.TokenNameFinderTrainerTool.TrainerToolParams;
 import opennlp.tools.cmdline.params.TrainingToolParams;
 import opennlp.tools.namefind.NameSample;
+import opennlp.tools.namefind.NameSampleTypeFilter;
 import opennlp.tools.namefind.TokenNameFinderModel;
 import opennlp.tools.util.InvalidFormatException;
 import opennlp.tools.util.model.ArtifactSerializer;
@@ -38,6 +40,8 @@ public final class TokenNameFinderTraine
     extends AbstractTrainerTool<NameSample, TrainerToolParams> {
   
   interface TrainerToolParams extends TrainingParams, TrainingToolParams {
+    @ParameterDescription(valueName = "types", description = "name types to use for training")
+    String getNameTypes();
   }
 
   public TokenNameFinderTrainerTool() {
@@ -162,6 +166,11 @@ public final class TokenNameFinderTraine
         
     CmdLineUtil.checkOutputFile("name finder model", modelOutFile);
 
+    if (params.getNameTypes() != null) {
+      String nameTypes[] = params.getNameTypes().split(",");
+      sampleStream = new NameSampleTypeFilter(nameTypes, sampleStream);
+    }
+    
     TokenNameFinderModel model;
     try {
       model = opennlp.tools.namefind.NameFinderME.train(

Added: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java?rev=1309371&view=auto
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java (added)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java Wed Apr  4 13:25:03 2012
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreemnets.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.namefind;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import opennlp.tools.util.FilterObjectStream;
+import opennlp.tools.util.ObjectStream;
+import opennlp.tools.util.Span;
+
+/**
+ * A stream which removes Name Samples which do not have a certain type.
+ */
+public class NameSampleTypeFilter extends FilterObjectStream<NameSample, NameSample> {
+
+  private final Set<String> types;
+
+  public NameSampleTypeFilter(String[] types, ObjectStream<NameSample> samples) {
+    super(samples);
+    this.types = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(types)));
+  }
+  
+  public NameSampleTypeFilter(Set<String> types, ObjectStream<NameSample> samples) {
+    super(samples);
+    this.types = Collections.unmodifiableSet(new HashSet<String>(types));
+  }
+
+  public NameSample read() throws IOException {
+    
+    NameSample sample = samples.read();
+    
+    if (sample != null) {
+      
+      List<Span> filteredNames = new ArrayList<Span>();
+      
+      for (Span name : sample.getNames()) {
+        if (types.contains(name.getType())) {
+          filteredNames.add(name);
+        }
+      }
+      
+      return new NameSample(sample.getSentence(),
+          filteredNames.toArray(new Span[filteredNames.size()]), sample.isClearAdaptiveDataSet());
+    }
+    else {
+      return null;
+    }
+  }
+}

Propchange: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/namefind/NameSampleTypeFilter.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain