You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2023/05/07 07:29:46 UTC
[opennlp] branch Add_tests_for_ModelLoader_classes_in_cmdline_sub-packages created (now 748e04ad)
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a change to branch Add_tests_for_ModelLoader_classes_in_cmdline_sub-packages
in repository https://gitbox.apache.org/repos/asf/opennlp.git
at 748e04ad Add tests for ModelLoader classes in cmdline sub-packages
This branch includes the following new commits:
new 748e04ad Add tests for ModelLoader classes in cmdline sub-packages
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[opennlp] 01/01: Add tests for ModelLoader classes in cmdline sub-packages
Posted by ma...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch Add_tests_for_ModelLoader_classes_in_cmdline_sub-packages
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 748e04ada2d6228cf6933b9301e6dcc4abb44e5a
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Sun May 7 09:29:37 2023 +0200
Add tests for ModelLoader classes in cmdline sub-packages
- adds several ...ModelLoaderTests
- extracts `EnabledWhenCDNAvailable` annotation into a separate file for better re-use
---
.../opennlp/tools/cmdline/GenerateManualTool.java | 3 +-
.../opennlp/tools/EnabledWhenCDNAvailable.java | 67 ++++++++++++++++++++
.../tools/cmdline/AbstractModelLoaderTest.java | 73 ++++++++++++++++++++++
.../cmdline/chunker/ChunkerModelLoaderTest.java | 64 +++++++++++++++++++
.../LanguageDetectorModelLoaderTest.java | 64 +++++++++++++++++++
.../namefind/TokenNameFinderModelLoaderTest.java | 64 +++++++++++++++++++
.../tools/cmdline/postag/POSModelLoaderTest.java | 67 ++++++++++++++++++++
.../sentdetect/SentenceModelLoaderTest.java | 67 ++++++++++++++++++++
.../tokenizer/TokenizerModelLoaderTest.java | 67 ++++++++++++++++++++
.../java/opennlp/tools/util/DownloadUtilTest.java | 43 +------------
10 files changed, 536 insertions(+), 43 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
index 4cda33dc..25fcc253 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
@@ -248,8 +248,7 @@ public class GenerateManualTool {
private static String splitLongLines(String stringBlock) {
StringBuilder sb = new StringBuilder();
String line;
- try {
- BufferedReader reader = new BufferedReader(new StringReader(stringBlock));
+ try (BufferedReader reader = new BufferedReader(new StringReader(stringBlock))) {
while ((line = reader.readLine()) != null) {
if (line.length() <= MAX_LINE_LENGTH) {
sb.append(line).append("\n");
diff --git a/opennlp-tools/src/test/java/opennlp/tools/EnabledWhenCDNAvailable.java b/opennlp-tools/src/test/java/opennlp/tools/EnabledWhenCDNAvailable.java
new file mode 100644
index 00000000..c3a49257
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/EnabledWhenCDNAvailable.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools;
+
+import java.io.IOException;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.ParameterizedTest;
+
+import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation;
+
+/**
+ * A custom JUnit5 conditional annotation which can be used to enable/disable tests at runtime.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(EnabledWhenCDNAvailable.CDNAvailableCondition.class)
+@ParameterizedTest
+public @interface EnabledWhenCDNAvailable {
+
+ String hostname();
+
+ int TIMEOUT_MS = 2000;
+
+ // JUnit5 execution condition to decide whether tests can assume CDN downloads are possible (= online).
+ class CDNAvailableCondition implements ExecutionCondition {
+
+ @Override
+ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+ final var optional = findAnnotation(context.getElement(), EnabledWhenCDNAvailable.class);
+ if (optional.isPresent()) {
+ final EnabledWhenCDNAvailable annotation = optional.get();
+ final String host = annotation.hostname();
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(host, 80), TIMEOUT_MS);
+ return ConditionEvaluationResult.enabled("CDN is reachable.");
+ } catch (IOException e) {
+ // Unreachable, unresolvable or timeout
+ return ConditionEvaluationResult.disabled("CDN is unreachable.");
+ }
+ }
+ return ConditionEvaluationResult.enabled("Nothing annotated with DisabledWhenOffline.");
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/AbstractModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/AbstractModelLoaderTest.java
new file mode 100644
index 00000000..5a511707
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/AbstractModelLoaderTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractModelLoaderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(AbstractModelLoaderTest.class);
+
+ private static final String BASE_URL_MODELS_V15 = "https://opennlp.sourceforge.net/models-1.5/";
+ private static final String BASE_URL_MODELS_V183 = "https://dlcdn.apache.org/opennlp/models/langdetect/1.8.3/";
+ protected static final Path OPENNLP_DIR = Paths.get(System.getProperty("user.home") + "/.opennlp/");
+
+ protected static void downloadVersion15Model(String modelName) throws IOException {
+ final URL url = new URL(BASE_URL_MODELS_V15 + modelName);
+ if (!Files.isDirectory(OPENNLP_DIR)) {
+ OPENNLP_DIR.toFile().mkdir();
+ }
+ final String filename = url.toString().substring(url.toString().lastIndexOf("/") + 1);
+ final Path localFile = Paths.get(OPENNLP_DIR.toString(), filename);
+
+ if (!Files.exists(localFile)) {
+ logger.debug("Downloading model from {} to {}.", url, localFile);
+ try (final InputStream in = new BufferedInputStream(url.openStream())) {
+ Files.copy(in, localFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.debug("Download complete.");
+ }
+ }
+
+ protected static void downloadVersion183Model(String modelName) throws IOException {
+ final URL url = new URL(BASE_URL_MODELS_V183 + modelName);
+ if (!Files.isDirectory(OPENNLP_DIR)) {
+ OPENNLP_DIR.toFile().mkdir();
+ }
+ final String filename = url.toString().substring(url.toString().lastIndexOf("/") + 1);
+ final Path localFile = Paths.get(OPENNLP_DIR.toString(), filename);
+
+ if (!Files.exists(localFile)) {
+ logger.debug("Downloading model from {} to {}.", url, localFile);
+ try (final InputStream in = new BufferedInputStream(url.openStream())) {
+ Files.copy(in, localFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.debug("Download complete.");
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/chunker/ChunkerModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/chunker/ChunkerModelLoaderTest.java
new file mode 100644
index 00000000..22eac3a6
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/chunker/ChunkerModelLoaderTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.chunker;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+
+public class ChunkerModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private ChunkerModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en");
+ resources.forEach(lang -> {
+ try {
+ downloadVersion15Model("en-chunker.bin");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new ChunkerModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" chunker model loading")
+ @ValueSource(strings = {"en-chunker.bin"})
+ @EnabledWhenCDNAvailable(hostname = "opennlp.sourceforge.net")
+ public void testLoadModelViaResource(String modelName) throws IOException {
+ ChunkerModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/langdetect/LanguageDetectorModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/langdetect/LanguageDetectorModelLoaderTest.java
new file mode 100644
index 00000000..54138873
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/langdetect/LanguageDetectorModelLoaderTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.langdetect;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.langdetect.LanguageDetectorModel;
+
+public class LanguageDetectorModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private LanguageDetectorModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en");
+ resources.forEach(lang -> {
+ try {
+ downloadVersion183Model("langdetect-183.bin");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new LanguageDetectorModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" language detector model loading")
+ @ValueSource(strings = {"langdetect-183.bin"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelViaResource(String modelName) throws IOException {
+ LanguageDetectorModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/namefind/TokenNameFinderModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/namefind/TokenNameFinderModelLoaderTest.java
new file mode 100644
index 00000000..1c68edf9
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/namefind/TokenNameFinderModelLoaderTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.namefind;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+public class TokenNameFinderModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private TokenNameFinderModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en");
+ resources.forEach(lang -> {
+ try {
+ downloadVersion15Model("en-ner-location.bin");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new TokenNameFinderModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" NER model loading")
+ @ValueSource(strings = {"en-ner-location.bin"})
+ @EnabledWhenCDNAvailable(hostname = "opennlp.sourceforge.net")
+ public void testLoadModelViaResource(String modelName) throws IOException {
+ TokenNameFinderModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/postag/POSModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/postag/POSModelLoaderTest.java
new file mode 100644
index 00000000..f108ed12
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/postag/POSModelLoaderTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.postag;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.postag.POSModel;
+import opennlp.tools.util.DownloadUtil;
+
+public class POSModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private POSModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en", "de");
+ resources.forEach(lang -> {
+ try {
+ DownloadUtil.downloadModel(lang,
+ DownloadUtil.ModelType.POS, POSModel.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new POSModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" POS model loading")
+ @ValueSource(strings = {"en-ud-ewt", "de-ud-gsd"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelByLanguage(String langModel) throws IOException {
+ String modelName = "opennlp-" + langModel + "-pos-1.0-1.9.3.bin";
+ POSModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/sentdetect/SentenceModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/sentdetect/SentenceModelLoaderTest.java
new file mode 100644
index 00000000..8de25ccc
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/sentdetect/SentenceModelLoaderTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.sentdetect;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.sentdetect.SentenceModel;
+import opennlp.tools.util.DownloadUtil;
+
+public class SentenceModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private SentenceModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en", "de");
+ resources.forEach(lang -> {
+ try {
+ DownloadUtil.downloadModel(lang,
+ DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new SentenceModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" sentence model loading")
+ @ValueSource(strings = {"en-ud-ewt", "de-ud-gsd"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelByLanguage(String langModel) throws IOException {
+ String modelName = "opennlp-" + langModel + "-sentence-1.0-1.9.3.bin";
+ SentenceModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/tokenizer/TokenizerModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/tokenizer/TokenizerModelLoaderTest.java
new file mode 100644
index 00000000..79c01eed
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/tokenizer/TokenizerModelLoaderTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.tokenizer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.util.DownloadUtil;
+
+public class TokenizerModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private TokenizerModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en", "de");
+ resources.forEach(lang -> {
+ try {
+ DownloadUtil.downloadModel(lang,
+ DownloadUtil.ModelType.TOKENIZER, TokenizerModel.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new TokenizerModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" tokenizer model loading")
+ @ValueSource(strings = {"en-ud-ewt", "de-ud-gsd"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelByLanguage(String langModel) throws IOException {
+ String modelName = "opennlp-" + langModel + "-tokens-1.0-1.9.3.bin";
+ TokenizerModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
index 14881815..1de5558b 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
@@ -18,8 +18,6 @@
package opennlp.tools.util;
import java.io.IOException;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
@@ -31,33 +29,27 @@ import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.extension.ConditionEvaluationResult;
-import org.junit.jupiter.api.extension.ExecutionCondition;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;
+import opennlp.tools.EnabledWhenCDNAvailable;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.TokenizerModel;
import static org.junit.jupiter.api.Assertions.fail;
-import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation;
public class DownloadUtilTest {
private static final String APACHE_CDN = "dlcdn.apache.org";
- private static final int TIMEOUT_MS = 2000;
-
@BeforeAll
public static void cleanupWhenOnline() {
boolean isOnline;
try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(APACHE_CDN, 80), TIMEOUT_MS);
+ socket.connect(new InetSocketAddress(APACHE_CDN, 80), EnabledWhenCDNAvailable.TIMEOUT_MS);
isOnline = true;
} catch (IOException e) {
// Unreachable, unresolvable or timeout
@@ -132,35 +124,4 @@ public class DownloadUtilTest {
Arguments.of("nl", DownloadUtil.available_models.get("nl").get(MT_TOKENIZER))
);
}
-
- // JUnit5 execution condition to decide whether tests can assume CDN downloads are possible (= online).
- private static class CDNAvailableCondition implements ExecutionCondition {
-
- @Override
- public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
- final var optional = findAnnotation(context.getElement(), EnabledWhenCDNAvailable.class);
- if (optional.isPresent()) {
- final EnabledWhenCDNAvailable annotation = optional.get();
- final String host = annotation.hostname();
- try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(host, 80), TIMEOUT_MS);
- return ConditionEvaluationResult.enabled("CDN is reachable.");
- } catch (IOException e) {
- // Unreachable, unresolvable or timeout
- return ConditionEvaluationResult.disabled("CDN is unreachable.");
- }
- }
- return ConditionEvaluationResult.enabled("Nothing annotated with DisabledWhenOffline.");
- }
- }
-
- // Custom JUnit5 conditional @Disabled.. annotation
- @Retention(RetentionPolicy.RUNTIME)
- @ExtendWith(CDNAvailableCondition.class)
- @ParameterizedTest
- public @interface EnabledWhenCDNAvailable {
-
- String hostname();
-
- }
}