(opennlp) branch main updated: OPENNLP-1523 - Use the snowball-data set to write language-specific stemmer eval tests

[rz...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

rzo1 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opennlp.git


The following commit(s) were added to refs/heads/main by this push:
     new 9b2d184e OPENNLP-1523 - Use the snowball-data set to write language-specific stemmer eval tests
9b2d184e is described below

commit 9b2d184e19e270c990ba9d1f6fa6e60bbbbaf2ce
Author: Richard Zowalla <ri...@hs-heilbronn.de>
AuthorDate: Thu Dec 14 09:06:47 2023 +0100

    OPENNLP-1523 - Use the snowball-data set to write language-specific stemmer eval tests
---
 .../opennlp/tools/eval/SnowballTokenizerEval.java  | 75 ++++++++++++++++++++++
 1 file changed, 75 insertions(+)

diff --git a/opennlp-tools/src/test/java/opennlp/tools/eval/SnowballTokenizerEval.java b/opennlp-tools/src/test/java/opennlp/tools/eval/SnowballTokenizerEval.java
new file mode 100644
index 00000000..6baa50fb
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/eval/SnowballTokenizerEval.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package opennlp.tools.eval;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Locale;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import opennlp.tools.stemmer.snowball.SnowballStemmer;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Eval tests for the {@link SnowballStemmer} class.
+ * <p>
+ * Uses the data set provided by <a href="https://github.com/snowballstem/snowball-data"></a>
+ * to test all languages available in OpenNLP.
+ * <p>
+ */
+public class SnowballTokenizerEval extends AbstractEvalTest {
+
+  @ParameterizedTest
+  @EnumSource(SnowballStemmer.ALGORITHM.class)
+  public void test(SnowballStemmer.ALGORITHM lang) throws IOException {
+
+    final List<String> vocabulary = getData(lang, "voc.txt");
+    final List<String> expectedOutputs = getData(lang, "output.txt");
+
+    assertEquals(vocabulary.size(), expectedOutputs.size(), "Expected equally sized lists.");
+    final SnowballStemmer stemmer = new SnowballStemmer(lang);
+
+    for (int i = 0; i < vocabulary.size(); i++) {
+
+      final String word = vocabulary.get(i);
+      final String stem = expectedOutputs.get(i);
+
+      assertEquals(stem, stemmer.stem(word));
+    }
+  }
+
+  private List<String> getData(SnowballStemmer.ALGORITHM lang, String name) throws IOException {
+    final Path expectedOutput = getSnowballDataLanguagePath(
+        getSnowballDataPath(), lang).resolve(name);
+    return Files.readAllLines(expectedOutput);
+  }
+
+  private Path getSnowballDataPath() throws FileNotFoundException {
+    return getOpennlpDataDir().toPath().resolve("snowball-data");
+  }
+
+  private Path getSnowballDataLanguagePath(Path root, SnowballStemmer.ALGORITHM lang) {
+    return root.resolve(lang.toString().toLowerCase(Locale.ROOT));
+  }
+
+}