svn commit: r1731151 - in /opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer: DictionaryLemmatizer.java SimpleLemmatizer.java

[ra...@apache.org]

Author: ragerri
Date: Thu Feb 18 21:17:13 2016
New Revision: 1731151

URL: http://svn.apache.org/viewvc?rev=1731151&view=rev
Log:
OPENNLP-760 modifying dictionary lemmatizer to use general lemmatizer interface

Removed:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/SimpleLemmatizer.java
Modified:
    opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java

Modified: opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
URL: http://svn.apache.org/viewvc/opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java?rev=1731151&r1=1731150&r2=1731151&view=diff
==============================================================================
--- opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java (original)
+++ opennlp/trunk/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java Thu Feb 18 21:17:13 2016
@@ -17,16 +17,99 @@
 
 package opennlp.tools.lemmatizer;
 
-public interface DictionaryLemmatizer {
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * Lemmatize by simple dictionary lookup into a hashmap built from a file
+ * containing, for each line, word\tablemma\tabpostag.
+ * @version 2014-07-08
+ */
+public class DictionaryLemmatizer implements Lemmatizer {
+
+  /**
+   * The hashmap containing the dictionary.
+   */
+  private final HashMap<List<String>, String> dictMap;
+
+  /**
+   * Construct a hashmap from the input tab separated dictionary.
+   * 
+   * The input file should have, for each line, word\tablemma\tabpostag
+   * 
+   * @param dictionary
+   *          the input dictionary via inputstream
+   */
+  public DictionaryLemmatizer(final InputStream dictionary) {
+    this.dictMap = new HashMap<List<String>, String>();
+    final BufferedReader breader = new BufferedReader(new InputStreamReader(
+        dictionary));
+    String line;
+    try {
+      while ((line = breader.readLine()) != null) {
+        final String[] elems = line.split("\t");
+        this.dictMap.put(Arrays.asList(elems[0], elems[2]), elems[1]);
+      }
+    } catch (final IOException e) {
+      e.printStackTrace();
+    }
+  }
 
   /**
-   * Returns the lemma of the specified word with the specified part-of-speech.
-   *
-   * @param word The word whose lemmas are desired.
-   * @param postag The part-of-speech of the specified word.
-   * @return The lemma of the specified word given the specified part-of-speech.
+   * Get the Map containing the dictionary.
+   * 
+   * @return dictMap the Map
    */
-  public String lemmatize(String word, String postag);
+  public HashMap<List<String>, String> getDictMap() {
+    return this.dictMap;
+  }
 
+  /**
+   * Get the dictionary keys (word and postag).
+   * 
+   * @param word
+   *          the surface form word
+   * @param postag
+   *          the assigned postag
+   * @return returns the dictionary keys
+   */
+  private List<String> getDictKeys(final String word, final String postag) {
+    final List<String> keys = new ArrayList<String>();
+    keys.addAll(Arrays.asList(word.toLowerCase(), postag));
+    return keys;
+  }
+  
+  public String[] lemmatize(final String[] tokens, final String[] postags) {
+    List<String> lemmas = new ArrayList<String>();
+    for (int i = 0; i < tokens.length; i++) {
+      lemmas.add(this.apply(tokens[i], postags[i])); 
+    }
+    return lemmas.toArray(new String[lemmas.size()]);
+  }
 
+  /**
+   * Lookup lemma in a dictionary. Outputs "O" if not found.
+   * @param word the token
+   * @param postag the postag
+   * @return the lemma
+   */
+  public String apply(final String word, final String postag) {
+    String lemma = null;
+    final List<String> keys = this.getDictKeys(word, postag);
+    // lookup lemma as value of the map
+    final String keyValue = this.dictMap.get(keys);
+    if (keyValue != null) {
+      lemma = keyValue;
+    } else {
+      lemma = "O";
+    }
+    return lemma;
+  }
 }
+