You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opennlp.apache.org by kr...@apache.org on 2026/10/04 09:02:30 UTC
(opennlp) branch fix/download-util-cached-lookup updated: DownloadUtil keeps a local copy of the model index
This is an automated email from the ASF dual-hosted git repository.
krickert pushed a commit to branch fix/download-util-cached-lookup
in repository https://gitbox.apache.org/repos/asf/opennlp.git
The following commit(s) were added to refs/heads/fix/download-util-cached-lookup by this push:
new 18d85eb3f DownloadUtil keeps a local copy of the model index
18d85eb3f is described below
commit 18d85eb3faaa16eeee93040bb077bd210d42be96
Author: Kristian Rickert <kr...@apache.org>
AuthorDate: Sun Oct 4 05:02:22 2026 -0400
DownloadUtil keeps a local copy of the model index
Looking up a model by language reads the model index from the model
host on every run, so a reset connection failed the lookup even when
the model itself was already downloaded. DownloadUtil now keeps a copy
of the last index it read in the download directory and uses it when
the index cannot be read. A failed read no longer gives a partial index.
Assisted-by: Claude Code 2.1.289
Co-authored-by: Kristian Rickert <kr...@apache.org>
Co-authored-by: Claude <no...@anthropic.com>
Reviewed-by: Kristian Rickert <kr...@apache.org>
Signed-off-by: Kristian Rickert <kr...@apache.org>
---
.../main/java/opennlp/tools/util/DownloadUtil.java | 124 ++++++++++++++++++++-
.../opennlp/tools/util/DownloadParserTest.java | 87 ++++++++++++++-
2 files changed, 206 insertions(+), 5 deletions(-)
diff --git a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
index a350e7659..3342240a1 100644
--- a/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
+++ b/opennlp-core/opennlp-runtime/src/main/java/opennlp/tools/util/DownloadUtil.java
@@ -26,6 +26,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
+import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -61,6 +62,8 @@ public class DownloadUtil {
System.getProperty("OPENNLP_DOWNLOAD_MODEL_PATH", "models/ud-models-1.3/");
private static final String OPENNLP_DOWNLOAD_HOME = "OPENNLP_DOWNLOAD_HOME";
private static final String CHECKSUM_EXTENSION = ".sha512";
+ private static final String INDEX_EXTENSION = ".index.html";
+ private static final String TEMP_EXTENSION = ".tmp";
private static Map<String, Map<ModelType, URL>> availableModels;
@@ -181,10 +184,18 @@ public class DownloadUtil {
}
}
+ /**
+ * Lists the models published in the model index. A copy of the last index read is kept in
+ * the download directory and used when the index cannot be read.
+ *
+ * @return The model URLs per language and {@link ModelType}. Empty if the index could not
+ * be read.
+ */
public static Map<String, Map<ModelType, URL>> getAvailableModels() {
if (availableModels == null) {
try {
- DownloadParser p = new DownloadParser(new URI(BASE_URL + MODEL_URI_PATH).toURL());
+ final DownloadParser p = new DownloadParser(new URI(BASE_URL + MODEL_URI_PATH).toURL(),
+ getDownloadHome().resolve(indexFileName(MODEL_URI_PATH)));
availableModels = p.getAvailableModels();
} catch (MalformedURLException | URISyntaxException e) {
throw new RuntimeException(e);
@@ -343,6 +354,21 @@ public class DownloadUtil {
}
}
+ /**
+ * Names the local copy of the model index after the last segment of the model path, for
+ * example {@code ud-models-1.3.index.html} for {@code models/ud-models-1.3/}.
+ *
+ * @param modelPath The model path, with or without trailing slashes.
+ * @return The file name.
+ */
+ static String indexFileName(String modelPath) {
+ String path = modelPath;
+ while (path.endsWith("/")) {
+ path = path.substring(0, path.length() - 1);
+ }
+ return path.substring(path.lastIndexOf('/') + 1) + INDEX_EXTENSION;
+ }
+
private static Path getDownloadHome() {
return Paths.get(System.getProperty(OPENNLP_DOWNLOAD_HOME,
System.getProperty("user.home"))).resolve(".opennlp");
@@ -356,15 +382,103 @@ public class DownloadUtil {
private static final char TAG_END = '>';
private final URL indexUrl;
+ private final Path localIndex;
+ /**
+ * Initializes a parser that always reads the index page from {@code indexUrl}.
+ *
+ * @param indexUrl The index page.
+ */
DownloadParser(URL indexUrl) {
+ this(indexUrl, null);
+ }
+
+ /**
+ * Initializes a parser that keeps a copy of the index page.
+ *
+ * @param indexUrl The index page.
+ * @param localIndex The file that keeps a copy of the index page, or {@code null} to
+ * keep no copy.
+ */
+ DownloadParser(URL indexUrl, Path localIndex) {
Objects.requireNonNull(indexUrl);
this.indexUrl = indexUrl;
+ this.localIndex = localIndex;
}
+ /**
+ * Lists the models of the index page. If the page lists models, it is stored as the
+ * local copy. Otherwise, for example if the page cannot be read, the local copy is used.
+ *
+ * @return The model URLs per language and {@link ModelType}. Empty if neither the page
+ * nor the local copy lists models.
+ * @throws MalformedURLException Thrown if a model URL is malformed.
+ * @throws URISyntaxException Thrown if a model URL is not a valid URI.
+ */
Map<String, Map<ModelType, URL>> getAvailableModels()
throws MalformedURLException, URISyntaxException {
- return toMap(extractLinks(fetchPageIndex()));
+ final String page = fetchPageIndex();
+ if (page != null) {
+ final Map<String, Map<ModelType, URL>> models = toMap(extractLinks(page));
+ if (!models.isEmpty()) {
+ storeLocalIndex(page);
+ return models;
+ }
+ }
+ final String local = readLocalIndex();
+ if (local == null) {
+ return new HashMap<>();
+ }
+ logger.info("Using the local copy of the page index at {}.", localIndex);
+ return toMap(extractLinks(local));
+ }
+
+ /**
+ * Reads the local copy of the index page.
+ *
+ * @return The page content, or {@code null} if there is no readable copy.
+ */
+ private String readLocalIndex() {
+ if (localIndex == null || !Files.isRegularFile(localIndex)) {
+ return null;
+ }
+ try {
+ return Files.readString(localIndex, StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ logger.warn("Could not read the local copy of the page index at {}.", localIndex, e);
+ return null;
+ }
+ }
+
+ /**
+ * Stores the index page as the local copy. The page is written to a temporary file that
+ * then replaces the copy, so a reader never sees a partly written copy. A failure is only
+ * logged.
+ *
+ * @param page The page content.
+ */
+ private void storeLocalIndex(String page) {
+ if (localIndex == null) {
+ return;
+ }
+ try {
+ final Path directory = localIndex.toAbsolutePath().getParent();
+ Files.createDirectories(directory);
+ final Path temporary =
+ Files.createTempFile(directory, localIndex.getFileName().toString(), TEMP_EXTENSION);
+ try {
+ Files.writeString(temporary, page, StandardCharsets.UTF_8);
+ try {
+ Files.move(temporary, localIndex, StandardCopyOption.ATOMIC_MOVE);
+ } catch (AtomicMoveNotSupportedException e) {
+ Files.move(temporary, localIndex, StandardCopyOption.REPLACE_EXISTING);
+ }
+ } finally {
+ Files.deleteIfExists(temporary);
+ }
+ } catch (IOException e) {
+ logger.warn("Could not store the local copy of the page index at {}.", localIndex, e);
+ }
}
/**
@@ -540,6 +654,11 @@ public class DownloadUtil {
result.putIfAbsent(locale, models);
}
+ /**
+ * Reads the index page from its URL, joining its lines.
+ *
+ * @return The page content, or {@code null} if the page could not be read completely.
+ */
private String fetchPageIndex() {
final StringBuilder html = new StringBuilder();
try (BufferedReader br = new BufferedReader(
@@ -550,6 +669,7 @@ public class DownloadUtil {
}
} catch (IOException e) {
logger.error("Could not read page index from {}", indexUrl, e);
+ return null;
}
return html.toString();
diff --git a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
index cbfe6d01a..e5e89a8e0 100644
--- a/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
+++ b/opennlp-tools/src/test/java/opennlp/tools/util/DownloadParserTest.java
@@ -17,33 +17,44 @@
package opennlp.tools.util;
+import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.stream.Stream;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import opennlp.tools.models.ModelType;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
public class DownloadParserTest {
+ private static final String INDEX = "opennlp/tools/util/index.html";
+ private static final int LANGUAGES = 36;
+
@ParameterizedTest(name = "Verify \"{0}\" available models")
@MethodSource(value = "expectedModels")
void testAvailableModels(String language, Map<ModelType, String> expectedModels) {
- final URL baseUrl = fromClasspath("opennlp/tools/util/index.html");
+ final URL baseUrl = fromClasspath(INDEX);
assertNotNull(baseUrl);
final DownloadUtil.DownloadParser downloadParser = new DownloadUtil.DownloadParser(baseUrl);
@@ -51,7 +62,7 @@ public class DownloadParserTest {
try {
Map<String, Map<ModelType, URL>> result = downloadParser.getAvailableModels();
assertNotNull(result);
- assertEquals(36, result.size());
+ assertEquals(LANGUAGES, result.size());
final Map<ModelType, URL> availableModels = result.get(language);
assertNotNull(availableModels);
@@ -119,7 +130,7 @@ public class DownloadParserTest {
@MethodSource("indexPages")
void testExtractLinks(String page, List<String> expected) {
final DownloadUtil.DownloadParser downloadParser =
- new DownloadUtil.DownloadParser(fromClasspath("opennlp/tools/util/index.html"));
+ new DownloadUtil.DownloadParser(fromClasspath(INDEX));
assertEquals(expected, downloadParser.extractLinks(page));
}
@@ -136,6 +147,76 @@ public class DownloadParserTest {
}
}
+ @Test
+ void testLocalIndexIsUsedIfTheUrlCannotBeRead(@TempDir Path home) throws Exception {
+ final Path localIndex = home.resolve("index.html");
+ Files.writeString(localIndex, indexPage(), StandardCharsets.UTF_8);
+ final URL unreachable = new URI("file:/this/does/not/exist/").toURL();
+
+ final Map<String, Map<ModelType, URL>> result =
+ new DownloadUtil.DownloadParser(unreachable, localIndex).getAvailableModels();
+
+ assertEquals(LANGUAGES, result.size());
+ assertEquals(unreachable + OPENNLP + "en-ud-ewt-" + MODEL_SENT + VER + BIN,
+ result.get("en").get(ModelType.SENTENCE_DETECTOR).toExternalForm());
+ }
+
+ @Test
+ void testIndexReadFromTheUrlIsStored(@TempDir Path home) throws Exception {
+ final Path localIndex = home.resolve("not/created/yet/index.html");
+
+ final Map<String, Map<ModelType, URL>> result = new DownloadUtil.DownloadParser(
+ fromClasspath(INDEX), localIndex).getAvailableModels();
+
+ assertEquals(LANGUAGES, result.size());
+ assertEquals(indexPage(), Files.readString(localIndex, StandardCharsets.UTF_8));
+ try (Stream<Path> files = Files.list(localIndex.getParent())) {
+ assertEquals(List.of(localIndex), files.toList());
+ }
+ }
+
+ @Test
+ void testIndexReadFromTheUrlReplacesTheLocalIndex(@TempDir Path home) throws Exception {
+ final Path localIndex = home.resolve("index.html");
+ Files.writeString(localIndex, "<html><body>Old</body></html>", StandardCharsets.UTF_8);
+
+ final Map<String, Map<ModelType, URL>> result = new DownloadUtil.DownloadParser(
+ fromClasspath(INDEX), localIndex).getAvailableModels();
+
+ assertEquals(LANGUAGES, result.size());
+ assertEquals(indexPage(), Files.readString(localIndex, StandardCharsets.UTF_8));
+ }
+
+ @Test
+ void testFailedReadStoresNothing(@TempDir Path home) throws Exception {
+ final Path localIndex = home.resolve("index.html");
+
+ final Map<String, Map<ModelType, URL>> result = new DownloadUtil.DownloadParser(
+ new URI("file:/this/does/not/exist").toURL(), localIndex).getAvailableModels();
+
+ assertTrue(result.isEmpty());
+ assertFalse(Files.exists(localIndex));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"models/ud-models-1.3/", "models/ud-models-1.3", "models/ud-models-1.3//",
+ "ud-models-1.3", "/ud-models-1.3/"})
+ void testIndexFileName(String modelPath) {
+ assertEquals("ud-models-1.3.index.html", DownloadUtil.indexFileName(modelPath));
+ }
+
+ /**
+ * Reads the test index page as the parser keeps it: its lines joined without separators.
+ *
+ * @return The page content.
+ * @throws IOException Thrown if the resource cannot be read.
+ * @throws URISyntaxException Thrown if the resource URL is not a valid URI.
+ */
+ private String indexPage() throws IOException, URISyntaxException {
+ return String.join("", Files.readAllLines(
+ Path.of(fromClasspath(INDEX).toURI()), StandardCharsets.UTF_8));
+ }
+
private URL fromClasspath(String file) {
return Thread.currentThread().getContextClassLoader().getResource(file);
}