[opennlp] branch master updated: OPENNLP-1320: Makes lemmatize of MorfologikLemmatizer thread-safe (v2) (#436)

[jz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jzemerick pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/opennlp.git


The following commit(s) were added to refs/heads/master by this push:
     new 6212a940 OPENNLP-1320: Makes lemmatize of MorfologikLemmatizer thread-safe (v2) (#436)
6212a940 is described below

commit 6212a940350b4f579d9d040591dac43293cb0b73
Author: Richard Zowalla <13...@users.noreply.github.com>
AuthorDate: Wed Nov 23 15:32:49 2022 +0100

    OPENNLP-1320: Makes lemmatize of MorfologikLemmatizer thread-safe (v2) (#436)
    
    * OPENNLP-1320: Makes lemmatize of MorfologikLemmatizer thread-safe
    
    Co-authored-by: Lucas Avanço <av...@stilingue.com.br>
---
 .../lemmatizer/MorfologikLemmatizer.java           | 10 ++++-----
 .../builder/POSDictionayBuilderTest.java           | 26 +++++++++++++++-------
 2 files changed, 22 insertions(+), 14 deletions(-)

diff --git a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
index 22f42821..f9597a9c 100644
--- a/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
+++ b/opennlp-morfologik-addon/src/main/java/opennlp/morfologik/lemmatizer/MorfologikLemmatizer.java
@@ -28,27 +28,25 @@ import java.util.Set;
 
 import morfologik.stemming.Dictionary;
 import morfologik.stemming.DictionaryLookup;
-import morfologik.stemming.IStemmer;
 import morfologik.stemming.WordData;
 
 import opennlp.tools.lemmatizer.Lemmatizer;
 
 public class MorfologikLemmatizer implements Lemmatizer {
 
-  private IStemmer dictLookup;
+  private final Dictionary dictionary;
 
   public MorfologikLemmatizer(Path dictionaryPath) throws IllegalArgumentException,
       IOException {
     this(Dictionary.read(dictionaryPath));
   }
 
-  public MorfologikLemmatizer(Dictionary dictionary) throws IllegalArgumentException,
-      IOException {
-    dictLookup = new DictionaryLookup(dictionary);
+  public MorfologikLemmatizer(Dictionary dictionary) throws IllegalArgumentException {
+    this.dictionary = dictionary;
   }
 
   private List<String> lemmatize(String word, String postag) {
-    List<WordData> dictMap = dictLookup.lookup(word.toLowerCase());
+    List<WordData> dictMap = new DictionaryLookup(dictionary).lookup(word.toLowerCase());
     Set<String> lemmas = new HashSet<>();
     for (WordData wordData : dictMap) {
       if (Objects.equals(postag, asString(wordData.getTag()))) {
diff --git a/opennlp-morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java b/opennlp-morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java
index 39bc2cb3..46b6da24 100644
--- a/opennlp-morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java
+++ b/opennlp-morfologik-addon/src/test/java/opennlp/morfologik/builder/POSDictionayBuilderTest.java
@@ -22,7 +22,9 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
-import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 
 import morfologik.stemming.DictionaryMetadata;
 
@@ -50,16 +52,15 @@ public class POSDictionayBuilderTest {
     return builder.build(tabFilePath);
   }
 
-  public static void main(String[] args) throws Exception {
-
+  @Test
+  public void testMultithread() throws Exception {
     // Part 1: compile a FSA lemma dictionary
     // we need the tabular dictionary. It is mandatory to have info
     //  file with same name, but .info extension
-    Path textLemmaDictionary = Paths.get(
-        "/Users/wcolen/git/opennlp/opennlp-morfologik-addon/src/test/resources/dictionaryWithLemma.txt");
 
     // this will build a binary dictionary located in compiledLemmaDictionary
-    Path compiledLemmaDictionary = new MorfologikDictionayBuilder().build(textLemmaDictionary);
+    Path compiledLemmaDictionary = new MorfologikDictionayBuilder().build(
+        Paths.get(POSDictionayBuilderTest.class.getResource("/dictionaryWithLemma.txt").getPath()));
 
     // Part 2: load a MorfologikLemmatizer and use it
     MorfologikLemmatizer lemmatizer = new MorfologikLemmatizer(compiledLemmaDictionary);
@@ -67,8 +68,17 @@ public class POSDictionayBuilderTest {
     String[] toks = {"casa", "casa"};
     String[] tags = {"NOUN", "V"};
 
-    String[] lemmas = lemmatizer.lemmatize(toks, tags);
-    System.out.println(Arrays.toString(lemmas)); // outputs [casa, casar]
+    Runnable runnable = () -> {
+      String[] lemmas = lemmatizer.lemmatize(toks, tags);
+      Assertions.assertEquals("casa", lemmas[0]);
+      Assertions.assertEquals("casar", lemmas[1]);
+    };
+    ExecutorService executorService = Executors.newFixedThreadPool(2);
+    for (int i = 0; i < 1000; i++) {
+      executorService.execute(runnable);
+    }
+    executorService.shutdown();
+    executorService.awaitTermination(1, TimeUnit.SECONDS);
   }
 
   @Test