You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by co...@apache.org on 2017/05/17 15:53:12 UTC
[46/50] opennlp git commit: OPENNLP-1061 Add functionality to
DictionaryLemmatizer to output several lemmas for a given word postag pair
OPENNLP-1061 Add functionality to DictionaryLemmatizer to output several lemmas for a given word postag pair
closes #202
Project: http://git-wip-us.apache.org/repos/asf/opennlp/repo
Commit: http://git-wip-us.apache.org/repos/asf/opennlp/commit/a00624cf
Tree: http://git-wip-us.apache.org/repos/asf/opennlp/tree/a00624cf
Diff: http://git-wip-us.apache.org/repos/asf/opennlp/diff/a00624cf
Branch: refs/heads/LangDetect
Commit: a00624cf27791193be74a610723a9a0b0980d23f
Parents: c434b3a
Author: Rodrigo Agerri <ra...@apache.org>
Authored: Tue May 16 12:35:22 2017 +0200
Committer: Jörn Kottmann <jo...@apache.org>
Committed: Tue May 16 15:32:37 2017 +0200
----------------------------------------------------------------------
.../tools/lemmatizer/DictionaryLemmatizer.java | 11 ++--
.../DictionaryLemmatizerMultiTest.java | 64 ++++++++++++++++++++
.../tools/lemmatizer/smalldictionarymulti.dict | 5 ++
3 files changed, 76 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/opennlp/blob/a00624cf/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
index 37d488c..97d6854 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/lemmatizer/DictionaryLemmatizer.java
@@ -29,7 +29,7 @@ import java.util.Map;
/**
* Lemmatize by simple dictionary lookup into a hashmap built from a file
- * containing, for each line, word\tablemma\tabpostag.
+ * containing, for each line, word\tabpostag\tablemma.
* @version 2014-07-08
*/
public class DictionaryLemmatizer implements Lemmatizer {
@@ -42,7 +42,9 @@ public class DictionaryLemmatizer implements Lemmatizer {
/**
* Construct a hashmap from the input tab separated dictionary.
*
- * The input file should have, for each line, word\tablemma\tabpostag
+ * The input file should have, for each line, word\tabpostag\tablemma.
+ * Alternatively, if multiple lemmas are possible for each word,postag pair,
+ * then the format should be word\tab\postag\tablemma01#lemma02#lemma03
*
* @param dictionary
* the input dictionary via inputstream
@@ -54,7 +56,8 @@ public class DictionaryLemmatizer implements Lemmatizer {
String line;
while ((line = breader.readLine()) != null) {
final String[] elems = line.split("\t");
- this.dictMap.put(Arrays.asList(elems[0], elems[1]), Arrays.asList(elems[2]));
+ final String[] lemmas = elems[2].split("#");
+ this.dictMap.put(Arrays.asList(elems[0], elems[1]), Arrays.asList(lemmas));
}
}
@@ -137,7 +140,7 @@ public class DictionaryLemmatizer implements Lemmatizer {
final List<String> keys = this.getDictKeys(word, postag);
// lookup lemma as value of the map
final List<String> keyValues = this.dictMap.get(keys);
- if (!keyValues.isEmpty()) {
+ if (keyValues != null && !keyValues.isEmpty()) {
lemmasList.addAll(keyValues);
} else {
lemmasList.add("O");
http://git-wip-us.apache.org/repos/asf/opennlp/blob/a00624cf/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/DictionaryLemmatizerMultiTest.java
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/DictionaryLemmatizerMultiTest.java b/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/DictionaryLemmatizerMultiTest.java
new file mode 100644
index 0000000..d29830b
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/lemmatizer/DictionaryLemmatizerMultiTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.lemmatizer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class DictionaryLemmatizerMultiTest {
+
+ private static DictionaryLemmatizer dictionaryLemmatizer;
+
+ @BeforeClass
+ public static void loadDictionary() throws Exception {
+ dictionaryLemmatizer = new DictionaryLemmatizer(
+ DictionaryLemmatizerTest.class.getResourceAsStream(
+ "/opennlp/tools/lemmatizer/smalldictionarymulti.dict")
+ );
+ }
+
+ @Test
+ public void testForNullPointerException() {
+ List<String> sentence = Arrays.asList("The","dogs","were","running","and","barking",
+ "down","the","street");
+ List<String> sentencePOS = Arrays.asList("DT","NNS","VBD","VBG","CC","VBG","RP","DT","NN");
+ List<List<String>> expectedLemmas = new ArrayList<>();
+ expectedLemmas.add(Arrays.asList("the"));
+ expectedLemmas.add(Arrays.asList("dog"));
+ expectedLemmas.add(Arrays.asList("is"));
+ expectedLemmas.add(Arrays.asList("run,run"));
+ expectedLemmas.add(Arrays.asList("and"));
+ expectedLemmas.add(Arrays.asList("bark,bark"));
+ expectedLemmas.add(Arrays.asList("down"));
+ expectedLemmas.add(Arrays.asList("the"));
+ expectedLemmas.add(Arrays.asList("street"));
+
+ List<List<String>> actualLemmas = dictionaryLemmatizer.lemmatize(sentence, sentencePOS);
+
+ for (int i = 0; i < sentence.size(); i++) {
+ // don't compare cases where the word is not in the dictionary...
+ if (!actualLemmas.get(0).get(0).equals("O"))
+ Assert.assertEquals(expectedLemmas.get(i), actualLemmas.get(i));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/opennlp/blob/a00624cf/opennlp-tools/src/test/resources/opennlp/tools/lemmatizer/smalldictionarymulti.dict
----------------------------------------------------------------------
diff --git a/opennlp-tools/src/test/resources/opennlp/tools/lemmatizer/smalldictionarymulti.dict b/opennlp-tools/src/test/resources/opennlp/tools/lemmatizer/smalldictionarymulti.dict
new file mode 100644
index 0000000..b650a0b
--- /dev/null
+++ b/opennlp-tools/src/test/resources/opennlp/tools/lemmatizer/smalldictionarymulti.dict
@@ -0,0 +1,5 @@
+barking VBG bark#bark
+dogs NNS dog
+running VBG run#run
+down RP down
+street NN street
\ No newline at end of file