You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by ma...@apache.org on 2023/05/07 07:29:47 UTC
[opennlp] 01/01: Add tests for ModelLoader classes in cmdline sub-packages
This is an automated email from the ASF dual-hosted git repository.
mawiesne pushed a commit to branch Add_tests_for_ModelLoader_classes_in_cmdline_sub-packages
in repository https://gitbox.apache.org/repos/asf/opennlp.git
commit 748e04ada2d6228cf6933b9301e6dcc4abb44e5a
Author: Martin Wiesner <ma...@hs-heilbronn.de>
AuthorDate: Sun May 7 09:29:37 2023 +0200
Add tests for ModelLoader classes in cmdline sub-packages
- adds several ...ModelLoaderTests
- extracts `EnabledWhenCDNAvailable` annotation into a separate file for better re-use
---
.../opennlp/tools/cmdline/GenerateManualTool.java | 3 +-
.../opennlp/tools/EnabledWhenCDNAvailable.java | 67 ++++++++++++++++++++
.../tools/cmdline/AbstractModelLoaderTest.java | 73 ++++++++++++++++++++++
.../cmdline/chunker/ChunkerModelLoaderTest.java | 64 +++++++++++++++++++
.../LanguageDetectorModelLoaderTest.java | 64 +++++++++++++++++++
.../namefind/TokenNameFinderModelLoaderTest.java | 64 +++++++++++++++++++
.../tools/cmdline/postag/POSModelLoaderTest.java | 67 ++++++++++++++++++++
.../sentdetect/SentenceModelLoaderTest.java | 67 ++++++++++++++++++++
.../tokenizer/TokenizerModelLoaderTest.java | 67 ++++++++++++++++++++
.../java/opennlp/tools/util/DownloadUtilTest.java | 43 +------------
10 files changed, 536 insertions(+), 43 deletions(-)
diff --git a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
index 4cda33dc..25fcc253 100644
--- a/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
+++ b/opennlp-tools/src/main/java/opennlp/tools/cmdline/GenerateManualTool.java
@@ -248,8 +248,7 @@ public class GenerateManualTool {
private static String splitLongLines(String stringBlock) {
StringBuilder sb = new StringBuilder();
String line;
- try {
- BufferedReader reader = new BufferedReader(new StringReader(stringBlock));
+ try (BufferedReader reader = new BufferedReader(new StringReader(stringBlock))) {
while ((line = reader.readLine()) != null) {
if (line.length() <= MAX_LINE_LENGTH) {
sb.append(line).append("\n");
diff --git a/opennlp-tools/src/test/java/opennlp/tools/EnabledWhenCDNAvailable.java b/opennlp-tools/src/test/java/opennlp/tools/EnabledWhenCDNAvailable.java
new file mode 100644
index 00000000..c3a49257
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/EnabledWhenCDNAvailable.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools;
+
+import java.io.IOException;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+
+import org.junit.jupiter.api.extension.ConditionEvaluationResult;
+import org.junit.jupiter.api.extension.ExecutionCondition;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.ExtensionContext;
+import org.junit.jupiter.params.ParameterizedTest;
+
+import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation;
+
+/**
+ * A custom JUnit5 conditional annotation which can be used to enable/disable tests at runtime.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@ExtendWith(EnabledWhenCDNAvailable.CDNAvailableCondition.class)
+@ParameterizedTest
+public @interface EnabledWhenCDNAvailable {
+
+ String hostname();
+
+ int TIMEOUT_MS = 2000;
+
+ // JUnit5 execution condition to decide whether tests can assume CDN downloads are possible (= online).
+ class CDNAvailableCondition implements ExecutionCondition {
+
+ @Override
+ public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
+ final var optional = findAnnotation(context.getElement(), EnabledWhenCDNAvailable.class);
+ if (optional.isPresent()) {
+ final EnabledWhenCDNAvailable annotation = optional.get();
+ final String host = annotation.hostname();
+ try (Socket socket = new Socket()) {
+ socket.connect(new InetSocketAddress(host, 80), TIMEOUT_MS);
+ return ConditionEvaluationResult.enabled("CDN is reachable.");
+ } catch (IOException e) {
+ // Unreachable, unresolvable or timeout
+ return ConditionEvaluationResult.disabled("CDN is unreachable.");
+ }
+ }
+ return ConditionEvaluationResult.enabled("Nothing annotated with DisabledWhenOffline.");
+ }
+ }
+
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/AbstractModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/AbstractModelLoaderTest.java
new file mode 100644
index 00000000..5a511707
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/AbstractModelLoaderTest.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public abstract class AbstractModelLoaderTest {
+
+ private static final Logger logger = LoggerFactory.getLogger(AbstractModelLoaderTest.class);
+
+ private static final String BASE_URL_MODELS_V15 = "https://opennlp.sourceforge.net/models-1.5/";
+ private static final String BASE_URL_MODELS_V183 = "https://dlcdn.apache.org/opennlp/models/langdetect/1.8.3/";
+ protected static final Path OPENNLP_DIR = Paths.get(System.getProperty("user.home") + "/.opennlp/");
+
+ protected static void downloadVersion15Model(String modelName) throws IOException {
+ final URL url = new URL(BASE_URL_MODELS_V15 + modelName);
+ if (!Files.isDirectory(OPENNLP_DIR)) {
+ OPENNLP_DIR.toFile().mkdir();
+ }
+ final String filename = url.toString().substring(url.toString().lastIndexOf("/") + 1);
+ final Path localFile = Paths.get(OPENNLP_DIR.toString(), filename);
+
+ if (!Files.exists(localFile)) {
+ logger.debug("Downloading model from {} to {}.", url, localFile);
+ try (final InputStream in = new BufferedInputStream(url.openStream())) {
+ Files.copy(in, localFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.debug("Download complete.");
+ }
+ }
+
+ protected static void downloadVersion183Model(String modelName) throws IOException {
+ final URL url = new URL(BASE_URL_MODELS_V183 + modelName);
+ if (!Files.isDirectory(OPENNLP_DIR)) {
+ OPENNLP_DIR.toFile().mkdir();
+ }
+ final String filename = url.toString().substring(url.toString().lastIndexOf("/") + 1);
+ final Path localFile = Paths.get(OPENNLP_DIR.toString(), filename);
+
+ if (!Files.exists(localFile)) {
+ logger.debug("Downloading model from {} to {}.", url, localFile);
+ try (final InputStream in = new BufferedInputStream(url.openStream())) {
+ Files.copy(in, localFile, StandardCopyOption.REPLACE_EXISTING);
+ }
+ logger.debug("Download complete.");
+ }
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/chunker/ChunkerModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/chunker/ChunkerModelLoaderTest.java
new file mode 100644
index 00000000..22eac3a6
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/chunker/ChunkerModelLoaderTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.chunker;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.chunker.ChunkerModel;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+
+public class ChunkerModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private ChunkerModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en");
+ resources.forEach(lang -> {
+ try {
+ downloadVersion15Model("en-chunker.bin");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new ChunkerModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" chunker model loading")
+ @ValueSource(strings = {"en-chunker.bin"})
+ @EnabledWhenCDNAvailable(hostname = "opennlp.sourceforge.net")
+ public void testLoadModelViaResource(String modelName) throws IOException {
+ ChunkerModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/langdetect/LanguageDetectorModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/langdetect/LanguageDetectorModelLoaderTest.java
new file mode 100644
index 00000000..54138873
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/langdetect/LanguageDetectorModelLoaderTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.langdetect;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.langdetect.LanguageDetectorModel;
+
+public class LanguageDetectorModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private LanguageDetectorModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en");
+ resources.forEach(lang -> {
+ try {
+ downloadVersion183Model("langdetect-183.bin");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new LanguageDetectorModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" language detector model loading")
+ @ValueSource(strings = {"langdetect-183.bin"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelViaResource(String modelName) throws IOException {
+ LanguageDetectorModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/namefind/TokenNameFinderModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/namefind/TokenNameFinderModelLoaderTest.java
new file mode 100644
index 00000000..1c68edf9
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/namefind/TokenNameFinderModelLoaderTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.namefind;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.namefind.TokenNameFinderModel;
+
+public class TokenNameFinderModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private TokenNameFinderModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en");
+ resources.forEach(lang -> {
+ try {
+ downloadVersion15Model("en-ner-location.bin");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new TokenNameFinderModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" NER model loading")
+ @ValueSource(strings = {"en-ner-location.bin"})
+ @EnabledWhenCDNAvailable(hostname = "opennlp.sourceforge.net")
+ public void testLoadModelViaResource(String modelName) throws IOException {
+ TokenNameFinderModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/postag/POSModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/postag/POSModelLoaderTest.java
new file mode 100644
index 00000000..f108ed12
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/postag/POSModelLoaderTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.postag;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.postag.POSModel;
+import opennlp.tools.util.DownloadUtil;
+
+public class POSModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private POSModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en", "de");
+ resources.forEach(lang -> {
+ try {
+ DownloadUtil.downloadModel(lang,
+ DownloadUtil.ModelType.POS, POSModel.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new POSModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" POS model loading")
+ @ValueSource(strings = {"en-ud-ewt", "de-ud-gsd"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelByLanguage(String langModel) throws IOException {
+ String modelName = "opennlp-" + langModel + "-pos-1.0-1.9.3.bin";
+ POSModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/sentdetect/SentenceModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/sentdetect/SentenceModelLoaderTest.java
new file mode 100644
index 00000000..8de25ccc
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/sentdetect/SentenceModelLoaderTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.sentdetect;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.sentdetect.SentenceModel;
+import opennlp.tools.util.DownloadUtil;
+
+public class SentenceModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private SentenceModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en", "de");
+ resources.forEach(lang -> {
+ try {
+ DownloadUtil.downloadModel(lang,
+ DownloadUtil.ModelType.SENTENCE_DETECTOR, SentenceModel.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new SentenceModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" sentence model loading")
+ @ValueSource(strings = {"en-ud-ewt", "de-ud-gsd"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelByLanguage(String langModel) throws IOException {
+ String modelName = "opennlp-" + langModel + "-sentence-1.0-1.9.3.bin";
+ SentenceModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/cmdline/tokenizer/TokenizerModelLoaderTest.java b/opennlp-tools/src/test/java/opennlp/tools/cmdline/tokenizer/TokenizerModelLoaderTest.java
new file mode 100644
index 00000000..79c01eed
--- /dev/null
+++ b/opennlp-tools/src/test/java/opennlp/tools/cmdline/tokenizer/TokenizerModelLoaderTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package opennlp.tools.cmdline.tokenizer;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.List;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import opennlp.tools.EnabledWhenCDNAvailable;
+import opennlp.tools.cmdline.AbstractModelLoaderTest;
+import opennlp.tools.tokenize.TokenizerModel;
+import opennlp.tools.util.DownloadUtil;
+
+public class TokenizerModelLoaderTest extends AbstractModelLoaderTest {
+
+ // SUT
+ private TokenizerModelLoader loader;
+
+ @BeforeAll
+ public static void initResources() {
+ List<String> resources = List.of("en", "de");
+ resources.forEach(lang -> {
+ try {
+ DownloadUtil.downloadModel(lang,
+ DownloadUtil.ModelType.TOKENIZER, TokenizerModel.class);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @BeforeEach
+ public void setup() {
+ loader = new TokenizerModelLoader();
+ }
+
+ @ParameterizedTest(name = "Verify \"{0}\" tokenizer model loading")
+ @ValueSource(strings = {"en-ud-ewt", "de-ud-gsd"})
+ @EnabledWhenCDNAvailable(hostname = "dlcdn.apache.org")
+ public void testLoadModelByLanguage(String langModel) throws IOException {
+ String modelName = "opennlp-" + langModel + "-tokens-1.0-1.9.3.bin";
+ TokenizerModel model = loader.loadModel(Files.newInputStream(OPENNLP_DIR.resolve(modelName)));
+ Assertions.assertNotNull(model);
+ Assertions.assertTrue(model.isLoadedFromSerialized());
+ }
+}
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
index 14881815..1de5558b 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadUtilTest.java
@@ -18,8 +18,6 @@
package opennlp.tools.util;
import java.io.IOException;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.net.URL;
@@ -31,33 +29,27 @@ import java.util.stream.Stream;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.extension.ConditionEvaluationResult;
-import org.junit.jupiter.api.extension.ExecutionCondition;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;
+import opennlp.tools.EnabledWhenCDNAvailable;
import opennlp.tools.sentdetect.SentenceModel;
import opennlp.tools.tokenize.TokenizerModel;
import static org.junit.jupiter.api.Assertions.fail;
-import static org.junit.platform.commons.util.AnnotationUtils.findAnnotation;
public class DownloadUtilTest {
private static final String APACHE_CDN = "dlcdn.apache.org";
- private static final int TIMEOUT_MS = 2000;
-
@BeforeAll
public static void cleanupWhenOnline() {
boolean isOnline;
try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(APACHE_CDN, 80), TIMEOUT_MS);
+ socket.connect(new InetSocketAddress(APACHE_CDN, 80), EnabledWhenCDNAvailable.TIMEOUT_MS);
isOnline = true;
} catch (IOException e) {
// Unreachable, unresolvable or timeout
@@ -132,35 +124,4 @@ public class DownloadUtilTest {
Arguments.of("nl", DownloadUtil.available_models.get("nl").get(MT_TOKENIZER))
);
}
-
- // JUnit5 execution condition to decide whether tests can assume CDN downloads are possible (= online).
- private static class CDNAvailableCondition implements ExecutionCondition {
-
- @Override
- public ConditionEvaluationResult evaluateExecutionCondition(ExtensionContext context) {
- final var optional = findAnnotation(context.getElement(), EnabledWhenCDNAvailable.class);
- if (optional.isPresent()) {
- final EnabledWhenCDNAvailable annotation = optional.get();
- final String host = annotation.hostname();
- try (Socket socket = new Socket()) {
- socket.connect(new InetSocketAddress(host, 80), TIMEOUT_MS);
- return ConditionEvaluationResult.enabled("CDN is reachable.");
- } catch (IOException e) {
- // Unreachable, unresolvable or timeout
- return ConditionEvaluationResult.disabled("CDN is unreachable.");
- }
- }
- return ConditionEvaluationResult.enabled("Nothing annotated with DisabledWhenOffline.");
- }
- }
-
- // Custom JUnit5 conditional @Disabled.. annotation
- @Retention(RetentionPolicy.RUNTIME)
- @ExtendWith(CDNAvailableCondition.class)
- @ParameterizedTest
- public @interface EnabledWhenCDNAvailable {
-
- String hostname();
-
- }
}